eval-unlearn 1.0.0__tar.gz

eval_unlearn-1.0.0/PKG-INFO

@@ -0,0 +1,229 @@
+Metadata-Version: 2.4
+Name: eval-unlearn
+Version: 1.0.0
+Summary: Unlearning Benchmark for Text-to-Image Models
+Author: Eval-Learn Team
+License: MIT
+Project-URL: Homepage, https://github.com/nikhilr2907/eval-unlearn
+Project-URL: Documentation, https://eval-unlearn.readthedocs.io
+Project-URL: Source, https://github.com/nikhilr2907/eval-unlearn
+Project-URL: Bug Tracker, https://github.com/nikhilr2907/eval-unlearn/issues
+Keywords: diffusion,unlearning,evaluation,text-to-image,benchmark,machine-learning
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=2.4.0
+Requires-Dist: scipy>=1.17.0
+Requires-Dist: tqdm>=4.66.5
+Requires-Dist: requests>=2.28.1
+Requires-Dist: safetensors>=0.7.0
+Requires-Dist: diffusers>=0.37.0
+Requires-Dist: huggingface_hub>=1.8.0
+Requires-Dist: Pillow>=12.1.0
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: torch>=2.11.0
+Requires-Dist: transformers>=5.3.0
+Requires-Dist: datasets>=4.8.0
+Requires-Dist: pyyaml>=5.1
+Provides-Extra: asr
+Requires-Dist: nudenet>=3.4.2; extra == "asr"
+Provides-Extra: fid
+Requires-Dist: torchvision>=0.26.0; extra == "fid"
+Provides-Extra: coco
+Requires-Dist: torchvision>=0.26.0; extra == "coco"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: ruff>=0.0.1; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: eval-unlearn[asr,coco,dev,fid]; extra == "all"
+
+# eval-unlearn
+
+A benchmarking framework for evaluating concept-unlearning techniques in text-to-image diffusion models.
+
+Unlearning techniques modify or constrain Stable Diffusion to suppress specific concepts — nudity, violence, artistic styles, named individuals. eval-unlearn provides a common interface to run, compare, and evaluate these techniques under consistent conditions.
+
+---
+
+## Techniques
+
+| Technique | Key |
+|-----------|-----|
+| Erased Stable Diffusion | `esd` |
+| Mass Concept Erasure | `mace` |
+| Unified Concept Editing | `uce` |
+| Selective Synaptic Dampening | `ssd` |
+| Concept Ablation | `ca` |
+| CoGFD | `cogfd` |
+| TraSCE | `trasce` |
+| SAFREE | `safree` |
+| Safe Latent Diffusion | `sld` |
+| AdvUnlearn | `advunlearn` |
+| Concept Steerers | `concept_steerers` |
+| SAeUron | `saeuron` |
+| Free Run (custom model) | `free_run` |
+
+## Metrics
+
+| Metric | Key | What it measures |
+|--------|-----|-----------------|
+| ASR — I2P | `asr_i2p` | Attack success rate on I2P prompts |
+| ASR — P4D | `asr_p4d` | Attack success rate via P4D adversarial prompts |
+| ASR — MMA Diffusion | `asr_mma_diffusion` | Attack success rate via MMA-Diffusion GCG attack |
+| ASR — Ring-A-Bell | `asr_ring_a_bell` | Attack success rate via genetic adversarial prompt discovery |
+| Erasure Retention Rate | `err` | Concept erasure vs. unrelated concept retention |
+| FID | `fid` | Image quality vs. COCO reference |
+| CLIP Score | `clip_score` | Prompt-image alignment |
+| UA-IRA | `ua_ira` | Unsafe concept alignment vs. retain concept alignment |
+| TIFA | `tifa` | Text-image faithfulness via VQA |
+
+---
+
+## Installation
+
+### 1. Install eval-unlearn
+
+```bash
+pip install eval-unlearn
+```
+
+### 2. Install technique packages
+
+Technique implementations are hosted on [Hugging Face](https://huggingface.co/datasets/Unlearningltd/Packages). Clone the repo once, pull LFS files, then install only what you need:
+
+```bash
+git clone https://huggingface.co/datasets/Unlearningltd/Packages
+cd Packages
+git lfs pull
+```
+
+```bash
+pip install -e esd/
+pip install -e mace/
+pip install -e uce/
+pip install -e ssd/
+pip install -e ca/
+pip install -e cogfd/
+pip install -e trasce/
+pip install -e saeuron/
+pip install -e safree/
+pip install -e concept-steerers/
+pip install -e advunlearn/
+```
+
+SLD is built into eval-unlearn via the `diffusers` library and requires no extra install.
+
+### 3. Install metric packages
+
+From the cloned `Packages` directory (see step 2 above):
+
+```bash
+pip install -e p4d/
+pip install -e mma_diff/
+pip install -e RING_A_BELL/
+pip install -e Q16/
+```
+
+```bash
+# NudeNet (nudity ASR)
+pip install "eval-unlearn[asr]"
+
+# FID / COCO metrics
+pip install "eval-unlearn[fid,coco]"
+```
+
+### 4. Hugging Face authentication
+
+Create a `.env` file in the directory you run `eval-unlearn run` from:
+
+```
+HF_TOKEN=your_token_here
+```
+
+---
+
+## Quick start
+
+Benchmarks are defined in a JSON or YAML config file:
+
+```json
+{
+  "output_dir": "results/esd_nudity",
+  "technique": {
+    "name": "esd",
+    "config": { "erase_concept": "nudity", "train_method": "noxattn", "device": "cuda" }
+  },
+  "metrics": [
+    { "name": "asr_i2p",    "config": { "concept_name": "nudity", "device": "cuda" } },
+    { "name": "fid",        "config": { "device": "cuda" } },
+    { "name": "clip_score", "config": { "device": "cuda" } }
+  ]
+}
+```
+
+Run it:
+
+```bash
+eval-unlearn run --config config.json
+```
+
+Results are written to `output_dir` as JSON.
+
+### Useful commands
+
+```bash
+eval-unlearn plugins   # list installed techniques and metrics
+eval-unlearn models    # show the base model each technique targets
+```
+
+---
+
+## Examples
+
+The [`examples/`](examples/) directory contains ready-to-run configs for all techniques across nudity and violence concepts:
+
+```
+examples/
+  nudity/     one config per technique (esd.json, mace.json, ...)
+  violence/   same, for violence concept
+  data/       seed prompts and concept vectors used by the configs
+```
+
+Run all nudity benchmarks in sequence:
+
+```bash
+python nudity_unlearning_demo.py
+```
+
+Run all violence benchmarks:
+
+```bash
+python nudity_unlearning_demo_violence.py
+```
+
+---
+
+## Documentation
+
+Full configuration reference, technique guides, metric descriptions, and experiment recipes:
+
+**https://eval-unlearn.readthedocs.io**
+
+Package on PyPI: **https://pypi.org/project/eval-unlearn/**
+
+Key pages:
+
+- [Getting started](docs/docs/getting-started.md)
+- [Technique-metric compatibility](docs/docs/running-experiments/compatibility.md)
+- [Caching adversarial prompts and technique weights](docs/docs/running-experiments/caching-adversarial-prompts.md)
+
+---
+
+## License
+
+MIT

eval_unlearn-1.0.0/README.md

@@ -0,0 +1,185 @@
+# eval-unlearn
+
+A benchmarking framework for evaluating concept-unlearning techniques in text-to-image diffusion models.
+
+Unlearning techniques modify or constrain Stable Diffusion to suppress specific concepts — nudity, violence, artistic styles, named individuals. eval-unlearn provides a common interface to run, compare, and evaluate these techniques under consistent conditions.
+
+---
+
+## Techniques
+
+| Technique | Key |
+|-----------|-----|
+| Erased Stable Diffusion | `esd` |
+| Mass Concept Erasure | `mace` |
+| Unified Concept Editing | `uce` |
+| Selective Synaptic Dampening | `ssd` |
+| Concept Ablation | `ca` |
+| CoGFD | `cogfd` |
+| TraSCE | `trasce` |
+| SAFREE | `safree` |
+| Safe Latent Diffusion | `sld` |
+| AdvUnlearn | `advunlearn` |
+| Concept Steerers | `concept_steerers` |
+| SAeUron | `saeuron` |
+| Free Run (custom model) | `free_run` |
+
+## Metrics
+
+| Metric | Key | What it measures |
+|--------|-----|-----------------|
+| ASR — I2P | `asr_i2p` | Attack success rate on I2P prompts |
+| ASR — P4D | `asr_p4d` | Attack success rate via P4D adversarial prompts |
+| ASR — MMA Diffusion | `asr_mma_diffusion` | Attack success rate via MMA-Diffusion GCG attack |
+| ASR — Ring-A-Bell | `asr_ring_a_bell` | Attack success rate via genetic adversarial prompt discovery |
+| Erasure Retention Rate | `err` | Concept erasure vs. unrelated concept retention |
+| FID | `fid` | Image quality vs. COCO reference |
+| CLIP Score | `clip_score` | Prompt-image alignment |
+| UA-IRA | `ua_ira` | Unsafe concept alignment vs. retain concept alignment |
+| TIFA | `tifa` | Text-image faithfulness via VQA |
+
+---
+
+## Installation
+
+### 1. Install eval-unlearn
+
+```bash
+pip install eval-unlearn
+```
+
+### 2. Install technique packages
+
+Technique implementations are hosted on [Hugging Face](https://huggingface.co/datasets/Unlearningltd/Packages). Clone the repo once, pull LFS files, then install only what you need:
+
+```bash
+git clone https://huggingface.co/datasets/Unlearningltd/Packages
+cd Packages
+git lfs pull
+```
+
+```bash
+pip install -e esd/
+pip install -e mace/
+pip install -e uce/
+pip install -e ssd/
+pip install -e ca/
+pip install -e cogfd/
+pip install -e trasce/
+pip install -e saeuron/
+pip install -e safree/
+pip install -e concept-steerers/
+pip install -e advunlearn/
+```
+
+SLD is built into eval-unlearn via the `diffusers` library and requires no extra install.
+
+### 3. Install metric packages
+
+From the cloned `Packages` directory (see step 2 above):
+
+```bash
+pip install -e p4d/
+pip install -e mma_diff/
+pip install -e RING_A_BELL/
+pip install -e Q16/
+```
+
+```bash
+# NudeNet (nudity ASR)
+pip install "eval-unlearn[asr]"
+
+# FID / COCO metrics
+pip install "eval-unlearn[fid,coco]"
+```
+
+### 4. Hugging Face authentication
+
+Create a `.env` file in the directory you run `eval-unlearn run` from:
+
+```
+HF_TOKEN=your_token_here
+```
+
+---
+
+## Quick start
+
+Benchmarks are defined in a JSON or YAML config file:
+
+```json
+{
+  "output_dir": "results/esd_nudity",
+  "technique": {
+    "name": "esd",
+    "config": { "erase_concept": "nudity", "train_method": "noxattn", "device": "cuda" }
+  },
+  "metrics": [
+    { "name": "asr_i2p",    "config": { "concept_name": "nudity", "device": "cuda" } },
+    { "name": "fid",        "config": { "device": "cuda" } },
+    { "name": "clip_score", "config": { "device": "cuda" } }
+  ]
+}
+```
+
+Run it:
+
+```bash
+eval-unlearn run --config config.json
+```
+
+Results are written to `output_dir` as JSON.
+
+### Useful commands
+
+```bash
+eval-unlearn plugins   # list installed techniques and metrics
+eval-unlearn models    # show the base model each technique targets
+```
+
+---
+
+## Examples
+
+The [`examples/`](examples/) directory contains ready-to-run configs for all techniques across nudity and violence concepts:
+
+```
+examples/
+  nudity/     one config per technique (esd.json, mace.json, ...)
+  violence/   same, for violence concept
+  data/       seed prompts and concept vectors used by the configs
+```
+
+Run all nudity benchmarks in sequence:
+
+```bash
+python nudity_unlearning_demo.py
+```
+
+Run all violence benchmarks:
+
+```bash
+python nudity_unlearning_demo_violence.py
+```
+
+---
+
+## Documentation
+
+Full configuration reference, technique guides, metric descriptions, and experiment recipes:
+
+**https://eval-unlearn.readthedocs.io**
+
+Package on PyPI: **https://pypi.org/project/eval-unlearn/**
+
+Key pages:
+
+- [Getting started](docs/docs/getting-started.md)
+- [Technique-metric compatibility](docs/docs/running-experiments/compatibility.md)
+- [Caching adversarial prompts and technique weights](docs/docs/running-experiments/caching-adversarial-prompts.md)
+
+---
+
+## License
+
+MIT

eval_unlearn-1.0.0/pyproject.toml

@@ -0,0 +1,120 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "eval-unlearn"
+version = "1.0.0"
+description = "Unlearning Benchmark for Text-to-Image Models"
+readme = {file = "README.md", content-type = "text/markdown"}
+requires-python = ">=3.11"
+license = {text = "MIT"}
+authors = [
+    {name = "Eval-Learn Team"}
+]
+keywords = ["diffusion", "unlearning", "evaluation", "text-to-image", "benchmark", "machine-learning"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "numpy>=2.4.0",
+    "scipy>=1.17.0",
+    "tqdm>=4.66.5",
+    "requests>=2.28.1",
+    "safetensors>=0.7.0",
+    "diffusers>=0.37.0",
+    "huggingface_hub>=1.8.0",
+    "Pillow>=12.1.0",
+    "python-dotenv>=1.2.2",
+    "torch>=2.11.0",
+    "transformers>=5.3.0",
+    "datasets>=4.8.0",
+    "pyyaml>=5.1",
+]
+
+[project.urls]
+Homepage = "https://github.com/nikhilr2907/eval-unlearn"
+Documentation = "https://eval-unlearn.readthedocs.io"
+Source = "https://github.com/nikhilr2907/eval-unlearn"
+"Bug Tracker" = "https://github.com/nikhilr2907/eval-unlearn/issues"
+
+[project.optional-dependencies]
+# Metric extras (PyPI packages)
+asr = ["nudenet>=3.4.2"]
+fid = ["torchvision>=0.26.0"]
+coco = ["torchvision>=0.26.0"]
+# Development
+dev = ["pytest>=7.0.0", "pytest-cov>=4.0.0", "ruff>=0.0.1", "mypy>=1.0.0"]
+all = ["eval-unlearn[asr,fid,coco,dev]"]
+
+# Note: Technique packages (esd, mace, uce, safree, sld, saeuron)
+# are local editable installs declared in requirements.txt
+
+[project.scripts]
+eval-unlearn = "eval_unlearn.cli:main"
+
+# Entry points for plugin discovery
+[project.entry-points."eval_unlearn.techniques"]
+free_run = "eval_unlearn.techniques.free_run.wrapper:FreeRunTechnique"
+esd = "eval_unlearn.techniques.esd.wrapper:ESDTechnique"
+mace = "eval_unlearn.techniques.mace.wrapper:MACETechnique"
+sld = "eval_unlearn.techniques.sld.wrapper:SLDTechnique"
+uce = "eval_unlearn.techniques.uce.wrapper:UCETechnique"
+safree = "eval_unlearn.techniques.SAFREE.wrapper:SAFREETechnique"
+concept_steerers = "eval_unlearn.techniques.concept_steerers.wrapper:ConceptSteerersTechnique"
+saeuron = "eval_unlearn.techniques.saeuron.wrapper:SAeUronTechnique"
+cogfd = "eval_unlearn.techniques.cogfd.wrapper:CoGFDTechnique"
+advunlearn = "eval_unlearn.techniques.advunlearn.wrapper:AdvUnlearnTechnique"
+ssd = "eval_unlearn.techniques.ssd.wrapper:SSDTechnique"
+trasce = "eval_unlearn.techniques.trasce.wrapper:TraSCETechnique"
+ca = "eval_unlearn.techniques.ca.wrapper:CATechnique"
+
+[project.entry-points."eval_unlearn.metrics"]
+asr_p4d = "eval_unlearn.metrics.asr_p4d.metric:ASRP4D"
+asr_i2p = "eval_unlearn.metrics.asr_i2p.metric:ASRMetric"
+asr_ring_a_bell = "eval_unlearn.metrics.asr_ring_a_bell.metric:ASRRingABellMetric"
+asr_mma_diffusion = "eval_unlearn.metrics.asr_mma_diffusion.metric:MMADiffusionMetric"
+fid = "eval_unlearn.metrics.fid.metric:FIDMetric"
+err = "eval_unlearn.metrics.err.metric:ERRMetric"
+tifa = "eval_unlearn.metrics.tifa.metric:TIFAMetric"
+clip_score = "eval_unlearn.metrics.clip_score.metric:CLIPScoreMetric"
+ua_ira = "eval_unlearn.metrics.ua_ira.metric:UAIRAMetric"
+
+[project.entry-points."eval_unlearn.datasets"]
+i2p_csv = "eval_unlearn.datasets.i2p_csv:load_i2p_csv"
+err_composite = "eval_unlearn.datasets.err_composite:load_err_composite"
+tifa_csv = "eval_unlearn.datasets.tifa_csv:load_tifa_csv"
+coco_parquet = "eval_unlearn.datasets.coco_parquet:load_coco_parquet"
+ua_ira_csv = "eval_unlearn.datasets.ua_ira_csv:load_ua_ira_csv"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+eval_unlearn = ["**/*.pth", "**/*.json", "**/*.yaml", "**/*.npy"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "integration: marks tests that require GPU and real models (deselect with '-m \"not integration\"')",
+]
+addopts = "--cov=eval_unlearn --cov-report=term-missing --cov-report=xml -p no:warnings"
+
+[tool.coverage.run]
+source = ["eval_unlearn"]
+omit = [
+    "*/tests/*",
+    "*/__pycache__/*",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+    "raise NotImplementedError",
+    "except ImportError",
+]
+show_missing = true

eval_unlearn-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

eval_unlearn-1.0.0/src/eval_unlearn/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

eval_unlearn-1.0.0/src/eval_unlearn/artifacts/__init__.py

@@ -0,0 +1,3 @@
+from .writer import ArtifactWriter
+
+__all__ = ["ArtifactWriter"]