biasid 1.0.0__tar.gz

biasid-1.0.0/PKG-INFO

@@ -0,0 +1,82 @@
+Metadata-Version: 2.3
+Name: biasid
+Version: 1.0.0
+Summary: biasid - a tool for analyzing transformation bias and robustness of AI-generated image detectors
+Requires-Dist: datasets>=4.8.0
+Requires-Dist: joblib>=1.5.3
+Requires-Dist: matplotlib>=3.10.0
+Requires-Dist: pandas>=3.0.0
+Requires-Dist: pillow>=12.1.0
+Requires-Dist: scienceplots>=2.2.1
+Requires-Dist: torch>=2.10.0
+Requires-Dist: torchmetrics>=1.9.0
+Requires-Dist: torchvision>=0.25.0
+Requires-Dist: tqdm>=4.67.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# `biasid` - A Tool for Analyzing Transformation Bias and Robustness of AI-Generated Image Detectors
+
+## Installation
+```
+pip install biasid
+```
+
+## Usage
+```python
+import matplotlib.pyplot as plt
+import torch
+from torch import nn
+from torchvision.transforms import v2
+
+from biasid import DetectorWrapper, compute_metrics, create_bias_plot, get_score_df
+from biasid.datasets import FolderDataset
+from biasid.transforms import JPEG
+
+
+# toy detector, use your own instead
+class MyDetector(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.layer = nn.Linear(3 * 32 * 32, 1)
+
+    def forward(self, images):
+        return self.layer(images.flatten(start_dim=1))
+
+
+if __name__ == "__main__":
+    my_detector = MyDetector()
+
+    # wrap your model into DetectorWrapper
+    detector = DetectorWrapper(
+        model=my_detector,
+        preprocessing=v2.CenterCrop(32),
+        postprocessing=torch.sigmoid,
+    )
+
+    # define datasets
+    real = FolderDataset(root="real")
+    fake = FolderDataset(root="fake")
+
+    # define a transformation
+    jpeg = JPEG()
+
+    # compute scores
+    score_df = get_score_df(
+        detector=detector,
+        real_dataset=real,
+        fake_dataset=fake,
+        output_dir="output/scores",
+        transform=jpeg,
+        batch_size=8,
+        num_workers=4,
+    )
+
+    # compute metrics (with bootstrapping)
+    metrics = compute_metrics(score_df)
+    metrics.save("output/metrics")
+
+    # visualize results
+    create_bias_plot(result=metrics, real_subset="real", fake_subset="fake")
+    plt.savefig("output/plots/jpeg.png")
+```

biasid-1.0.0/README.md

@@ -0,0 +1,65 @@
+# `biasid` - A Tool for Analyzing Transformation Bias and Robustness of AI-Generated Image Detectors
+
+## Installation
+```
+pip install biasid
+```
+
+## Usage
+```python
+import matplotlib.pyplot as plt
+import torch
+from torch import nn
+from torchvision.transforms import v2
+
+from biasid import DetectorWrapper, compute_metrics, create_bias_plot, get_score_df
+from biasid.datasets import FolderDataset
+from biasid.transforms import JPEG
+
+
+# toy detector, use your own instead
+class MyDetector(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.layer = nn.Linear(3 * 32 * 32, 1)
+
+    def forward(self, images):
+        return self.layer(images.flatten(start_dim=1))
+
+
+if __name__ == "__main__":
+    my_detector = MyDetector()
+
+    # wrap your model into DetectorWrapper
+    detector = DetectorWrapper(
+        model=my_detector,
+        preprocessing=v2.CenterCrop(32),
+        postprocessing=torch.sigmoid,
+    )
+
+    # define datasets
+    real = FolderDataset(root="real")
+    fake = FolderDataset(root="fake")
+
+    # define a transformation
+    jpeg = JPEG()
+
+    # compute scores
+    score_df = get_score_df(
+        detector=detector,
+        real_dataset=real,
+        fake_dataset=fake,
+        output_dir="output/scores",
+        transform=jpeg,
+        batch_size=8,
+        num_workers=4,
+    )
+
+    # compute metrics (with bootstrapping)
+    metrics = compute_metrics(score_df)
+    metrics.save("output/metrics")
+
+    # visualize results
+    create_bias_plot(result=metrics, real_subset="real", fake_subset="fake")
+    plt.savefig("output/plots/jpeg.png")
+```

biasid-1.0.0/pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "biasid"
+version = "1.0.0"
+description = "biasid - a tool for analyzing transformation bias and robustness of AI-generated image detectors"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "datasets>=4.8.0",
+    "joblib>=1.5.3",
+    "matplotlib>=3.10.0",
+    "pandas>=3.0.0",
+    "pillow>=12.1.0",
+    "scienceplots>=2.2.1",
+    "torch>=2.10.0",
+    "torchmetrics>=1.9.0",
+    "torchvision>=0.25.0",
+    "tqdm>=4.67.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.10.12,<0.11.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1",
+    "pytest-cov>=7.1.0",
+    "ruff>=0.16.2",
+]
+
+[tool.ruff]
+target-version = "py312"
+fix = true
+line-length = 120
+
+[tool.ruff.lint]
+extend-select = ["I"]
+extend-ignore = ["E402"]

biasid-1.0.0/src/biasid/__init__.py

@@ -0,0 +1,4 @@
+from biasid.core import get_score_df as get_score_df
+from biasid.detector_wrapper import DetectorWrapper as DetectorWrapper
+from biasid.metrics import compute_metrics as compute_metrics
+from biasid.visualization import create_bias_plot as create_bias_plot

biasid-1.0.0/src/biasid/core.py

@@ -0,0 +1,102 @@
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+from torch import Tensor
+from torch.utils.data import Dataset
+from tqdm.auto import tqdm
+
+from biasid.datasets import Metadata
+from biasid.detector_wrapper import DetectorWrapper
+from biasid.prediction import get_scores
+from biasid.transforms import LevelTransform
+
+
+def get_score_df(
+    detector: DetectorWrapper,
+    real_dataset: Dataset,
+    fake_dataset: Dataset,
+    output_dir: str | Path,
+    transform: LevelTransform | None = None,
+    batch_size: int = 1,
+    num_workers: int = 0,
+) -> pd.DataFrame:
+    """
+    Compute detection scores for real and fake images, clean and transformed.
+    All results are saved and automatically loaded during repeated calls.
+    """
+    output_dir = Path(output_dir)
+
+    rows: list[dict[str, Any]] = []
+
+    for label, dataset in zip(["real", "fake"], [real_dataset, fake_dataset]):
+        if len(dataset) == 0:
+            raise ValueError(f"Dataset {dataset!r} is empty.")
+        dataset_id = dataset.id if hasattr(dataset, "id") else label
+
+        _write_status(dataset=dataset_id, detector=detector.id)
+        scores, metadata = get_scores(
+            detector=detector,
+            dataset=dataset,
+            batch_size=batch_size,
+            num_workers=num_workers,
+            score_file=output_dir / dataset_id / "clean.pt",
+        )
+        rows.extend(
+            _expand_prediction_rows(
+                label=label,
+                transform_name="clean",
+                level=None,
+                level_label=transform.clean_level_label if transform is not None else None,
+                metadata=metadata,
+                scores=scores,
+            )
+        )
+
+        if transform is not None:
+            for level in transform.levels:
+                _write_status(dataset=dataset_id, detector=detector.id, transform=transform.id, level=level)
+                scores, metadata = get_scores(
+                    detector=detector,
+                    dataset=dataset,
+                    transform=transform.get_transform(level=level),
+                    batch_size=batch_size,
+                    num_workers=num_workers,
+                    score_file=output_dir / dataset_id / transform.id / f"{level}.pt",
+                )
+                rows.extend(
+                    _expand_prediction_rows(
+                        label=label,
+                        transform_name=transform.id,
+                        level=level,
+                        level_label=transform.get_level_label(level),
+                        metadata=metadata,
+                        scores=scores,
+                    )
+                )
+
+    return pd.DataFrame(rows)
+
+
+def _write_status(dataset: str, detector: str, transform: str | None = None, level: Any = None) -> None:
+    transform_description = f"{transform} @ {level}" if transform is not None else "clean"
+    tqdm.write(f"Predicting {dataset} ({transform_description}) with {detector}", end="")
+
+
+def _expand_prediction_rows(
+    scores: Tensor,
+    metadata: list[Metadata],
+    label: str,
+    transform_name: str,
+    level: Any,
+    level_label: str | None,
+) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for score, sample_metadata in zip(scores, metadata):
+        row: dict[str, Any] = {"label": label, **sample_metadata}
+        row["transform"] = transform_name
+        row["level"] = level
+        row["level_label"] = level_label
+        row["score"] = score.item()
+        rows.append(row)
+    return rows

biasid-1.0.0/src/biasid/datasets.py

@@ -0,0 +1,120 @@
+from collections.abc import Callable
+from pathlib import Path
+from typing import Literal
+
+import torch
+import torchvision.transforms.v2.functional as TF
+from datasets import load_dataset
+from PIL import Image
+from torch import Tensor
+from torch.utils.data import Dataset
+from torchvision.datasets.folder import IMG_EXTENSIONS
+
+ScalarMetadataValue = str | int | float | bool | None
+Metadata = dict[str, ScalarMetadataValue]
+
+
+class FolderDataset(Dataset):
+    """
+    Load images from a (nested) folder.
+    Metadata consists of the image's filename as 'image_id' and the parent folder name as 'subset'.
+    """
+
+    def __init__(self, root: str | Path) -> None:
+        self.root = Path(root)
+        self.paths = sorted([str(path) for path in self.root.rglob("*") if path.suffix.lower() in IMG_EXTENSIONS])
+
+    @property
+    def id(self) -> str:
+        return self.root.name
+
+    def __len__(self) -> int:
+        return len(self.paths)
+
+    def __getitem__(self, index: int) -> tuple[Tensor, Metadata]:
+        path = self.paths[index]
+        with open(path, "rb") as f:
+            img = Image.open(f).convert("RGB")
+        img = TF.to_dtype(TF.pil_to_tensor(img), dtype=torch.float, scale=True)
+
+        return img, {"image_id": Path(path).stem, "subset": Path(path).parent.name}
+
+
+class SynthCLIC(Dataset):
+    """
+    Load the SynthCLIC dataset (https://huggingface.co/datasets/marco-willi/synthclic).
+    """
+
+    def __init__(
+        self,
+        label: Literal["real", "fake"],
+        split: Literal["train", "validation", "test", "all"],
+    ) -> None:
+        self.label = label
+        self.split = split
+        self.ds = load_dataset(path="marco-willi/synthclic", split=split)
+
+        if label == "real":
+            self.ds = self.ds.filter(function=lambda _label: _label == 0, input_columns="label")
+        elif label == "fake":
+            self.ds = self.ds.filter(function=lambda _label: _label == 1, input_columns="label")
+        else:
+            raise ValueError(f"Invalid label {label!r}. Must be 'real' or 'fake'.")
+
+    @property
+    def id(self) -> str:
+        if self.label == "real":
+            return f"clic2020_{self.split}"
+        else:
+            return f"synthclic_{self.split}"
+
+    def __len__(self):
+        return len(self.ds)
+
+    def __getitem__(self, index) -> tuple[Tensor, Metadata]:
+        row = self.ds[index]
+        image = TF.to_dtype(TF.pil_to_tensor(row["image"].convert("RGB")), dtype=torch.float, scale=True)
+        metadata = {"image_id": row["image_id"], "subset": row["source"]}
+
+        return image, metadata
+
+
+class LabelDataset(Dataset):
+    """
+    Assign a fixed label to a dataset. Does not pass-through metadata.
+    """
+
+    def __init__(self, dataset: Dataset, label: int) -> None:
+        self.dataset = dataset
+        self.label = label
+
+    def __len__(self) -> int:
+        return len(self.dataset)  # type: ignore
+
+    def __getitem__(self, index) -> tuple[Tensor, int]:
+        sample = self.dataset[index]
+        if isinstance(sample, tuple):
+            sample = self.dataset[index][0]
+        return sample, self.label
+
+
+class TransformDataset(Dataset):
+    """
+    Apply a transform to a dataset. Metadata is preserved.
+    """
+
+    def __init__(self, dataset: Dataset, transform: Callable) -> None:
+        self.dataset = dataset
+        self.transform = transform
+
+    def __len__(self) -> int:
+        return len(self.dataset)  # type: ignore
+
+    def __getitem__(self, index: int) -> tuple[Tensor, Metadata]:
+        data = self.dataset[index]
+        if isinstance(data, tuple):
+            image, metadata = data
+        else:
+            image = data
+            metadata = {}
+        return self.transform(image), metadata

biasid-1.0.0/src/biasid/detector_wrapper.py

@@ -0,0 +1,35 @@
+from collections.abc import Callable
+
+from torch import Tensor, nn
+
+from biasid.transforms import IdentityTransform
+
+
+class DetectorWrapper(nn.Module):
+    """
+    Wrapper class for detectors.
+    Preprocessing is applied to the image before call to 'forward', but after any transformation.
+    Postprocessing is applied to the model's outputs.
+    """
+
+    def __init__(
+        self,
+        model: nn.Module,
+        preprocessing: Callable | None = None,
+        postprocessing: Callable | None = None,
+    ) -> None:
+        super().__init__()
+
+        self.model = model
+        self.preprocessing = preprocessing if preprocessing is not None else IdentityTransform()
+        self.postprocessing = postprocessing if postprocessing is not None else IdentityTransform()
+
+        self.device = next(self.model.parameters()).device
+        self.model.eval()
+
+    def forward(self, x: Tensor) -> Tensor:
+        return self.model(x)
+
+    @property
+    def id(self) -> str:
+        return self.model.__class__.__name__.lower()