rsdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rsdiff/__init__.py +3 -0
- rsdiff/cascade.py +184 -0
- rsdiff/cli.py +137 -0
- rsdiff/config.py +107 -0
- rsdiff/configs/rsdiff1.5.yaml +54 -0
- rsdiff/configs/rsdiff1.yaml +61 -0
- rsdiff/datasets/__init__.py +16 -0
- rsdiff/datasets/base.py +44 -0
- rsdiff/datasets/rsicd.py +92 -0
- rsdiff/eval/__init__.py +11 -0
- rsdiff/eval/clip.py +90 -0
- rsdiff/eval/fid.py +125 -0
- rsdiff-0.1.0.dist-info/METADATA +245 -0
- rsdiff-0.1.0.dist-info/RECORD +18 -0
- rsdiff-0.1.0.dist-info/WHEEL +4 -0
- rsdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rsdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rsdiff-0.1.0.dist-info/licenses/NOTICE +23 -0
rsdiff/datasets/rsicd.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
from PIL import Image
|
|
8
|
+
|
|
9
|
+
from rsdiff.datasets.base import RSDataset, Sample
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class RSICD(RSDataset):
|
|
13
|
+
"""Remote Sensing Image Captioning Dataset.
|
|
14
|
+
|
|
15
|
+
Expects the standard layout::
|
|
16
|
+
|
|
17
|
+
<root>/RSICD_images/*.jpg
|
|
18
|
+
<root>/dataset_rsicd.csv
|
|
19
|
+
|
|
20
|
+
The CSV has columns ``filename, split, imgid, sent1..sent5, label``.
|
|
21
|
+
A random one of the five captions is returned per ``__getitem__`` call
|
|
22
|
+
by default to act as caption-level augmentation; set ``caption_idx``
|
|
23
|
+
to a value in ``0..4`` to pin a specific column.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
name = "rsicd"
|
|
27
|
+
splits = ("train", "val", "test")
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
root: str | Path,
|
|
32
|
+
split: str = "train",
|
|
33
|
+
transform=None,
|
|
34
|
+
caption_idx: int | None = None,
|
|
35
|
+
images_subdir: str = "RSICD_images",
|
|
36
|
+
csv_name: str = "dataset_rsicd.csv",
|
|
37
|
+
) -> None:
|
|
38
|
+
super().__init__(root, split=split, transform=transform)
|
|
39
|
+
self.caption_idx = caption_idx
|
|
40
|
+
self.images_dir = self.root / images_subdir
|
|
41
|
+
csv_path = self.root / csv_name
|
|
42
|
+
if not csv_path.is_file():
|
|
43
|
+
raise FileNotFoundError(f"RSICD CSV not found at {csv_path}")
|
|
44
|
+
df = pd.read_csv(csv_path)
|
|
45
|
+
self.df = df[df["split"] == split].reset_index(drop=True)
|
|
46
|
+
self._caption_cols = [f"sent{i}" for i in range(1, 6)]
|
|
47
|
+
|
|
48
|
+
def __len__(self) -> int:
|
|
49
|
+
return len(self.df)
|
|
50
|
+
|
|
51
|
+
def __getitem__(self, idx: int) -> Sample:
|
|
52
|
+
row = self.df.iloc[idx]
|
|
53
|
+
img = Image.open(self.images_dir / row["filename"]).convert("RGB")
|
|
54
|
+
if self.transform is not None:
|
|
55
|
+
img = self.transform(img)
|
|
56
|
+
|
|
57
|
+
if self.caption_idx is None:
|
|
58
|
+
caption = row[random.choice(self._caption_cols)]
|
|
59
|
+
else:
|
|
60
|
+
caption = row[self._caption_cols[self.caption_idx]]
|
|
61
|
+
|
|
62
|
+
return Sample(
|
|
63
|
+
image=img,
|
|
64
|
+
caption=str(caption),
|
|
65
|
+
label=str(row.get("label", "")) or None,
|
|
66
|
+
meta={"filename": row["filename"], "imgid": int(row["imgid"])},
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def classes(self) -> list[str]:
|
|
71
|
+
return sorted(self.df["label"].dropna().unique().tolist())
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def sample_captions(
|
|
75
|
+
root: str | Path,
|
|
76
|
+
split: str = "test",
|
|
77
|
+
n: int | None = None,
|
|
78
|
+
seed: int = 17,
|
|
79
|
+
csv_name: str = "dataset_rsicd.csv",
|
|
80
|
+
) -> list[str]:
|
|
81
|
+
"""Pick ``n`` first captions (``sent1``) from ``split``, shuffled with ``seed``.
|
|
82
|
+
|
|
83
|
+
``n=None`` returns every row of the split. With ``seed=17``, the full test split and a
|
|
84
|
+
CSV built by ``scripts/build_rsicd_csv.py --split-csv``, this returns exactly the
|
|
85
|
+
captions of the published 1,093-image evaluation bundle.
|
|
86
|
+
"""
|
|
87
|
+
df = pd.read_csv(Path(root) / csv_name)
|
|
88
|
+
df = df[df["split"] == split].reset_index(drop=True)
|
|
89
|
+
if df.empty:
|
|
90
|
+
raise ValueError(f"no rows for split={split!r} in {Path(root) / csv_name}")
|
|
91
|
+
k = len(df) if n is None else min(n, len(df))
|
|
92
|
+
return df.sample(n=k, random_state=seed)["sent1"].astype(str).tolist()
|
rsdiff/eval/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Evaluation metrics for generated remote-sensing imagery.
|
|
2
|
+
|
|
3
|
+
- ``fid``: Fréchet Inception Distance (torchmetrics, Inception feature 2048 by default);
|
|
4
|
+
the metric behind the published FID 65.70.
|
|
5
|
+
- ``clip_score``: caption-image CLIP similarity with a shuffled-caption baseline.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from rsdiff.eval.clip import CLIPResult, clip_score
|
|
9
|
+
from rsdiff.eval.fid import FIDResult, fid
|
|
10
|
+
|
|
11
|
+
__all__ = ["CLIPResult", "FIDResult", "clip_score", "fid"]
|
rsdiff/eval/clip.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""CLIP score of a generation bundle, with a shuffled-caption null baseline.
|
|
2
|
+
|
|
3
|
+
The score is the mean cosine similarity between each image and its caption under OpenAI
|
|
4
|
+
CLIP ViT-B/32. The baseline pairs every image with another image's caption (fixed shuffle,
|
|
5
|
+
seed 17); ``delta`` is how much the real pairing beats chance. Published release values:
|
|
6
|
+
0.278 vs 0.232 (delta +0.046) on the 1,093-image RSICD test bundle.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import random
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import torch
|
|
16
|
+
from PIL import Image
|
|
17
|
+
|
|
18
|
+
from rsdiff.eval.fid import _auto_device
|
|
19
|
+
|
|
20
|
+
DEFAULT_CLIP_MODEL = "openai/clip-vit-base-patch32"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class CLIPResult:
|
|
25
|
+
clip_score: float
|
|
26
|
+
clip_score_std: float
|
|
27
|
+
clip_score_shuffled: float
|
|
28
|
+
delta: float
|
|
29
|
+
n: int
|
|
30
|
+
model: str
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def bundle_pairs(gen_dir: str | Path) -> tuple[list[str], list[Path]]:
|
|
34
|
+
"""Return the captions and PNGs of a bundle written by ``rsdiff sample``.
|
|
35
|
+
|
|
36
|
+
Captions come from ``captions.txt``; PNGs are paired with them in sorted file order.
|
|
37
|
+
"""
|
|
38
|
+
gen_dir = Path(gen_dir)
|
|
39
|
+
captions = (gen_dir / "captions.txt").read_text(encoding="utf-8").splitlines()
|
|
40
|
+
pngs = sorted(p for p in gen_dir.glob("*.png") if p.name != "_grid.png")
|
|
41
|
+
if len(pngs) != len(captions):
|
|
42
|
+
raise ValueError(f"{len(pngs)} PNGs but {len(captions)} captions in {gen_dir}")
|
|
43
|
+
return captions, pngs
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def clip_score(
|
|
47
|
+
gen_dir: str | Path,
|
|
48
|
+
model_id: str = DEFAULT_CLIP_MODEL,
|
|
49
|
+
shuffle_seed: int = 17,
|
|
50
|
+
device: str | None = None,
|
|
51
|
+
) -> CLIPResult:
|
|
52
|
+
from transformers import CLIPModel, CLIPProcessor
|
|
53
|
+
|
|
54
|
+
captions, pngs = bundle_pairs(gen_dir)
|
|
55
|
+
device = device or _auto_device()
|
|
56
|
+
model = CLIPModel.from_pretrained(model_id, use_safetensors=True).to(device).eval()
|
|
57
|
+
proc = CLIPProcessor.from_pretrained(model_id)
|
|
58
|
+
|
|
59
|
+
with torch.no_grad():
|
|
60
|
+
text = []
|
|
61
|
+
for i in range(0, len(captions), 64):
|
|
62
|
+
tk = proc(
|
|
63
|
+
text=captions[i : i + 64], return_tensors="pt", padding=True, truncation=True
|
|
64
|
+
).to(device)
|
|
65
|
+
text.append(model.get_text_features(**tk).cpu())
|
|
66
|
+
image = []
|
|
67
|
+
for i in range(0, len(pngs), 32):
|
|
68
|
+
imgs = [Image.open(p).convert("RGB") for p in pngs[i : i + 32]]
|
|
69
|
+
image.append(
|
|
70
|
+
model.get_image_features(**proc(images=imgs, return_tensors="pt").to(device)).cpu()
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
te = torch.cat(text)
|
|
74
|
+
te = te / te.norm(dim=-1, keepdim=True)
|
|
75
|
+
ie = torch.cat(image)
|
|
76
|
+
ie = ie / ie.norm(dim=-1, keepdim=True)
|
|
77
|
+
sim = (te * ie).sum(-1)
|
|
78
|
+
|
|
79
|
+
order = list(range(len(captions)))
|
|
80
|
+
random.Random(shuffle_seed).shuffle(order)
|
|
81
|
+
sim_shuffled = (te[order] * ie).sum(-1)
|
|
82
|
+
|
|
83
|
+
return CLIPResult(
|
|
84
|
+
clip_score=sim.mean().item(),
|
|
85
|
+
clip_score_std=sim.std().item(),
|
|
86
|
+
clip_score_shuffled=sim_shuffled.mean().item(),
|
|
87
|
+
delta=(sim.mean() - sim_shuffled.mean()).item(),
|
|
88
|
+
n=len(pngs),
|
|
89
|
+
model=model_id,
|
|
90
|
+
)
|
rsdiff/eval/fid.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Fréchet Inception Distance for generated RS imagery.
|
|
2
|
+
|
|
3
|
+
torchmetrics' ``FrechetInceptionDistance`` (InceptionV3 backbone). Stage-agnostic:
|
|
4
|
+
point ``gen_dir`` at whatever the model produced — LR 128² grids, SR 256² grids,
|
|
5
|
+
or full-cascade 256² — and ``real_*`` at the matching RSICD split. Both sides are
|
|
6
|
+
resized to ``image_size`` before the metric (Inception rescales to 299 internally),
|
|
7
|
+
so keep ``image_size`` equal to the stage you are scoring (128 for LR, 256 for SR /
|
|
8
|
+
cascade) and compare only against numbers computed the same way.
|
|
9
|
+
|
|
10
|
+
The thesis FID snippet (commented out in the legacy non-cascade DDPM script) used
|
|
11
|
+
``feature=64``; standard, paper-comparable FID is ``feature=2048``. Both are exposed
|
|
12
|
+
so we can reproduce the thesis number *and* report a comparable one.
|
|
13
|
+
|
|
14
|
+
Note: FID is biased for small samples. Use the full test split (~1093 generations),
|
|
15
|
+
not a 16-image grid, for a number worth quoting.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
import pandas as pd
|
|
24
|
+
import torch
|
|
25
|
+
from PIL import Image
|
|
26
|
+
from tqdm import tqdm
|
|
27
|
+
|
|
28
|
+
_IMG_EXTS = {".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff", ".webp"}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _auto_device() -> str:
|
|
32
|
+
# Not MPS: torchmetrics keeps FID statistics in float64, which MPS does not support.
|
|
33
|
+
return "cuda" if torch.cuda.is_available() else "cpu"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class FIDResult:
|
|
38
|
+
fid: float
|
|
39
|
+
feature: int
|
|
40
|
+
image_size: int
|
|
41
|
+
n_real: int
|
|
42
|
+
n_gen: int
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _load_uint8(path: Path, size: int) -> torch.Tensor:
|
|
46
|
+
"""Load one image as a CxHxW uint8 tensor at (size, size)."""
|
|
47
|
+
img = Image.open(path).convert("RGB").resize((size, size), Image.BICUBIC)
|
|
48
|
+
t = torch.frombuffer(img.tobytes(), dtype=torch.uint8)
|
|
49
|
+
return t.view(size, size, 3).permute(2, 0, 1).contiguous()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _real_paths(real_csv: Path | None, real_dir: Path, split: str) -> list[Path]:
|
|
53
|
+
if real_csv is not None:
|
|
54
|
+
df = pd.read_csv(real_csv)
|
|
55
|
+
df = df[df["split"] == split].reset_index(drop=True)
|
|
56
|
+
if df.empty:
|
|
57
|
+
raise ValueError(f"No rows for split={split!r} in {real_csv}")
|
|
58
|
+
return [real_dir / fn for fn in df["filename"]]
|
|
59
|
+
return [p for p in sorted(real_dir.iterdir()) if p.suffix.lower() in _IMG_EXTS]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _gen_paths(gen_dir: Path) -> list[Path]:
|
|
63
|
+
return sorted(p for p in gen_dir.rglob("*") if p.suffix.lower() in _IMG_EXTS)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def fid(
|
|
67
|
+
gen_dir: str | Path,
|
|
68
|
+
real_csv: str | Path | None = None,
|
|
69
|
+
real_dir: str | Path | None = None,
|
|
70
|
+
split: str = "test",
|
|
71
|
+
feature: int = 2048,
|
|
72
|
+
image_size: int = 256,
|
|
73
|
+
batch_size: int = 32,
|
|
74
|
+
max_n: int | None = None,
|
|
75
|
+
device: str | None = None,
|
|
76
|
+
) -> FIDResult:
|
|
77
|
+
from torchmetrics.image.fid import FrechetInceptionDistance
|
|
78
|
+
|
|
79
|
+
device = device or _auto_device()
|
|
80
|
+
gen_dir = Path(gen_dir)
|
|
81
|
+
if real_dir is None and real_csv is None:
|
|
82
|
+
raise ValueError("provide real_csv (+ real_dir) or real_dir")
|
|
83
|
+
real_dir = Path(real_dir) if real_dir is not None else None
|
|
84
|
+
real_csv = Path(real_csv) if real_csv is not None else None
|
|
85
|
+
if real_csv is not None and real_dir is None:
|
|
86
|
+
raise ValueError("real_csv needs real_dir (image root for its filenames)")
|
|
87
|
+
|
|
88
|
+
real = _real_paths(real_csv, real_dir, split)
|
|
89
|
+
gen = _gen_paths(gen_dir)
|
|
90
|
+
if max_n is not None:
|
|
91
|
+
real, gen = real[:max_n], gen[:max_n]
|
|
92
|
+
if not real or not gen:
|
|
93
|
+
raise ValueError(f"empty set (real={len(real)}, gen={len(gen)})")
|
|
94
|
+
|
|
95
|
+
metric = FrechetInceptionDistance(feature=feature, normalize=False).to(device)
|
|
96
|
+
|
|
97
|
+
def _feed(paths: list[Path], real_flag: bool, desc: str) -> int:
|
|
98
|
+
buf: list[torch.Tensor] = []
|
|
99
|
+
n = 0
|
|
100
|
+
for p in tqdm(paths, desc=desc):
|
|
101
|
+
if not p.exists():
|
|
102
|
+
continue
|
|
103
|
+
try:
|
|
104
|
+
buf.append(_load_uint8(p, image_size))
|
|
105
|
+
except Exception:
|
|
106
|
+
continue
|
|
107
|
+
if len(buf) >= batch_size:
|
|
108
|
+
metric.update(torch.stack(buf).to(device), real=real_flag)
|
|
109
|
+
n += len(buf)
|
|
110
|
+
buf.clear()
|
|
111
|
+
if buf:
|
|
112
|
+
metric.update(torch.stack(buf).to(device), real=real_flag)
|
|
113
|
+
n += len(buf)
|
|
114
|
+
return n
|
|
115
|
+
|
|
116
|
+
n_real = _feed(real, True, "fid/real")
|
|
117
|
+
n_gen = _feed(gen, False, "fid/gen")
|
|
118
|
+
|
|
119
|
+
return FIDResult(
|
|
120
|
+
fid=float(metric.compute().item()),
|
|
121
|
+
feature=feature,
|
|
122
|
+
image_size=image_size,
|
|
123
|
+
n_real=n_real,
|
|
124
|
+
n_gen=n_gen,
|
|
125
|
+
)
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: rsdiff
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Text-to-satellite-image diffusion: the RSDiff cascade (NCAA 2024), released weights, and its FID/CLIP evaluation.
|
|
5
|
+
Project-URL: Homepage, https://github.com/asebaq/rsdiff
|
|
6
|
+
Project-URL: Documentation, https://asebaq.github.io/rsdiff
|
|
7
|
+
Project-URL: Issues, https://github.com/asebaq/rsdiff/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/asebaq/rsdiff/blob/main/CHANGELOG.md
|
|
9
|
+
Project-URL: Paper, https://doi.org/10.1007/s00521-024-10363-3
|
|
10
|
+
Project-URL: Weights, https://huggingface.co/asebaq/rsdiff-sr-cascade-ep650
|
|
11
|
+
Author-email: Ahmad Sebaq <ahmad.sebaq@gmail.com>
|
|
12
|
+
License-Expression: Apache-2.0
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
License-File: NOTICE
|
|
15
|
+
Keywords: diffusion,imagen,remote-sensing,rsicd,satellite,text-to-image
|
|
16
|
+
Classifier: Development Status :: 3 - Alpha
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
25
|
+
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Requires-Dist: huggingface-hub>=0.25
|
|
28
|
+
Requires-Dist: imagen-pytorch==2.1.0
|
|
29
|
+
Requires-Dist: numpy>=1.26
|
|
30
|
+
Requires-Dist: omegaconf>=2.3
|
|
31
|
+
Requires-Dist: pandas>=2.2
|
|
32
|
+
Requires-Dist: pillow>=10.4
|
|
33
|
+
Requires-Dist: torch>=2.4
|
|
34
|
+
Requires-Dist: torchvision>=0.19
|
|
35
|
+
Requires-Dist: tqdm>=4.66
|
|
36
|
+
Requires-Dist: transformers<5,>=4.44
|
|
37
|
+
Provides-Extra: eval
|
|
38
|
+
Requires-Dist: torchmetrics[image]>=1.4; extra == 'eval'
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
<h1 align="center">
|
|
42
|
+
<a href="https://github.com/asebaq/rsdiff">
|
|
43
|
+
<picture>
|
|
44
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/asebaq/rsdiff/main/docs/assets/cover_dark.svg">
|
|
45
|
+
<img alt="rsdiff" src="https://raw.githubusercontent.com/asebaq/rsdiff/main/docs/assets/cover_light.svg" width="640">
|
|
46
|
+
</picture>
|
|
47
|
+
</a>
|
|
48
|
+
</h1>
|
|
49
|
+
|
|
50
|
+
<p align="center">
|
|
51
|
+
<a href="https://pypi.org/project/rsdiff/"><img alt="PyPI" src="https://img.shields.io/pypi/v/rsdiff?color=1D4ED8"></a>
|
|
52
|
+
<a href="https://pypi.org/project/rsdiff/"><img alt="Python" src="https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2Fasebaq%2Frsdiff%2Fmain%2Fpyproject.toml"></a>
|
|
53
|
+
<a href="https://github.com/asebaq/rsdiff/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/asebaq/rsdiff/actions/workflows/ci.yml/badge.svg"></a>
|
|
54
|
+
<a href="https://github.com/asebaq/rsdiff/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/github/license/asebaq/rsdiff"></a>
|
|
55
|
+
<a href="https://doi.org/10.1007/s00521-024-10363-3"><img alt="DOI" src="https://img.shields.io/badge/DOI-10.1007%2Fs00521--024--10363--3-blue"></a>
|
|
56
|
+
<a href="https://huggingface.co/asebaq/rsdiff-sr-cascade-ep650"><img alt="Hugging Face model" src="https://img.shields.io/badge/Hugging%20Face-rsdiff--sr--cascade--ep650-yellow"></a>
|
|
57
|
+
</p>
|
|
58
|
+
|
|
59
|
+
<p align="center">
|
|
60
|
+
<a href="#install"><strong>Install</strong></a> ·
|
|
61
|
+
<a href="#use"><strong>Use</strong></a> ·
|
|
62
|
+
<a href="#results"><strong>Results</strong></a> ·
|
|
63
|
+
<a href="#how-it-works"><strong>How it works</strong></a> ·
|
|
64
|
+
<a href="#limitations"><strong>Limitations</strong></a> ·
|
|
65
|
+
<a href="#training"><strong>Training</strong></a> ·
|
|
66
|
+
<a href="#roadmap"><strong>Roadmap</strong></a> ·
|
|
67
|
+
<a href="#citation"><strong>Citation</strong></a>
|
|
68
|
+
</p>
|
|
69
|
+
|
|
70
|
+
**Text-to-satellite-image diffusion, packaged.** rsdiff is the reference implementation
|
|
71
|
+
of RSDiff ([Neural Computing and Applications, 2024](https://doi.org/10.1007/s00521-024-10363-3)):
|
|
72
|
+
a T5-conditioned, Imagen-style cascaded diffusion model that turns a caption such as
|
|
73
|
+
"many buildings and green trees are around a playground ." into a 256x256 synthetic
|
|
74
|
+
remote-sensing image. One command downloads the released RSICD-trained checkpoint,
|
|
75
|
+
verifies it, and samples; a second scores the result with the same FID and CLIP-score
|
|
76
|
+
protocol used for the published numbers (FID 65.70, CLIP-score 0.278 on the full
|
|
77
|
+
RSICD test split).
|
|
78
|
+
|
|
79
|
+
Every image rsdiff produces is synthetic. It is not an observation or a prediction of
|
|
80
|
+
any real place, and each PNG says so in its metadata.
|
|
81
|
+
|
|
82
|
+
<p align="center">
|
|
83
|
+
<img src="https://raw.githubusercontent.com/asebaq/rsdiff/main/docs/figures/sample_montage.png" width="780" alt="Synthetic 256x256 samples generated from RSICD test captions">
|
|
84
|
+
</p>
|
|
85
|
+
|
|
86
|
+
## Install
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
pip install rsdiff # sampling + CLIP-score
|
|
90
|
+
pip install "rsdiff[eval]" # adds torchmetrics for FID
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Python 3.10+. Sampling runs on CUDA when available and otherwise on CPU. Apple MPS is
|
|
94
|
+
not supported: imagen-pytorch 2.1.0 and the torchmetrics FID both use float64.
|
|
95
|
+
|
|
96
|
+
## Use
|
|
97
|
+
|
|
98
|
+
Generate images from your own captions. The first run downloads the released checkpoint
|
|
99
|
+
(1.9 GB, pinned Hugging Face revision) and verifies its SHA-256:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
rsdiff sample --prompt "many buildings and green trees are around a playground ." -n 4 --out-dir generated
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
This writes 256x256 PNGs and a `captions.txt` to `generated/`. `--prompt` can be
|
|
106
|
+
repeated. Each PNG carries `Software`, `Description` (a synthetic-content notice), and
|
|
107
|
+
`Prompt` text chunks.
|
|
108
|
+
|
|
109
|
+
Or draw captions from RSICD (first caption, `sent1`, shuffled with `--seed`, default 17):
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
rsdiff sample --rsicd data/RSICD_optimal --split test -n 16 --out-dir generated
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Useful options: `--cond-scale` (default 5), `--batch-size` (default 2), `--lr-only`
|
|
116
|
+
(stop at the 128x128 base stage), `--cpu`, `--config rsdiff1.5|rsdiff1|path.yaml`, and
|
|
117
|
+
`--checkpoint PATH` (required for non-default configs). An interrupted run resumes:
|
|
118
|
+
batches whose PNGs already exist are skipped.
|
|
119
|
+
|
|
120
|
+
Score a generated directory against the RSICD test split:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
rsdiff eval fid generated --rsicd data/RSICD_optimal # --feature 2048 --size 256 by default
|
|
124
|
+
rsdiff eval clip generated # clip_score, clip_score_shuffled, delta
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Both print JSON. The same pieces are available from Python:
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from rsdiff.cascade import download_release, generate, load_cascade
|
|
131
|
+
from rsdiff.config import load_preset
|
|
132
|
+
from rsdiff.eval import clip_score
|
|
133
|
+
|
|
134
|
+
cfg = load_preset("rsdiff1.5")
|
|
135
|
+
trainer = load_cascade(cfg, download_release())
|
|
136
|
+
paths = generate(trainer.sample, ["a river runs through the green farmland ."], "generated")
|
|
137
|
+
print(clip_score("generated"))
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
`rsdiff.datasets.rsicd.sample_captions` reproduces the CLI's RSICD caption draw, and
|
|
141
|
+
`rsdiff.eval.fid` computes FID.
|
|
142
|
+
|
|
143
|
+
## Results
|
|
144
|
+
|
|
145
|
+
Released checkpoint, full RSICD test split (N=1,093), cascade output at 256x256,
|
|
146
|
+
`cond_scale=5`:
|
|
147
|
+
|
|
148
|
+
| Metric | Value |
|
|
149
|
+
| --- | --- |
|
|
150
|
+
| FID (torchmetrics Inception, feature 2048, both sides at 256) | **65.70** |
|
|
151
|
+
| CLIP-score (OpenAI CLIP ViT-B/32) | **0.278** |
|
|
152
|
+
| CLIP-score, shuffled-caption baseline | 0.232 |
|
|
153
|
+
| Parameters (base UNet 27.2 M + SR UNet 92.7 M) | 119.9 M[^params] |
|
|
154
|
+
|
|
155
|
+
[^params]: 119.9 M counts the two standalone UNets, as in the paper and model card.
|
|
156
|
+
Loaded inside the cascade, the SR UNet gains low-resolution conditioning layers
|
|
157
|
+
(105.0 M), so the cascade holds 132.2 M trainable parameters, plus the frozen T5-base
|
|
158
|
+
text encoder.
|
|
159
|
+
|
|
160
|
+
The CLIP-score sits 0.046 above the shuffled-caption baseline: the images follow their
|
|
161
|
+
captions, not just the RSICD domain.
|
|
162
|
+
|
|
163
|
+
| Curve | Figure |
|
|
164
|
+
| --- | --- |
|
|
165
|
+
| SR FID versus epoch (ep150 to ep1000) | [`docs/figures/sr_fid_curve.png`](docs/figures/sr_fid_curve.png) |
|
|
166
|
+
| CFG `cond_scale` sweep on the selected SR milestone | [`docs/figures/cfg_sweep.png`](docs/figures/cfg_sweep.png) |
|
|
167
|
+
| LR base FID versus epoch | [`docs/figures/lr_fid_curve.png`](docs/figures/lr_fid_curve.png) |
|
|
168
|
+
|
|
169
|
+
The numbers behind every figure are committed in [`results/`](results/). Methodology and
|
|
170
|
+
discussion: [`docs/REPORT.md`](docs/REPORT.md). Model card:
|
|
171
|
+
[`docs/model_card.md`](docs/model_card.md).
|
|
172
|
+
|
|
173
|
+
## How it works
|
|
174
|
+
|
|
175
|
+
1. A frozen **T5-base** encoder turns the caption into token embeddings.
|
|
176
|
+
2. A **128x128 base UNet** denoises from noise over 1,000 DDPM steps, cross-attending to
|
|
177
|
+
the text.
|
|
178
|
+
3. A **256x256 super-resolution UNet** denoises again, conditioned on the text and the
|
|
179
|
+
upsampled base image.
|
|
180
|
+
|
|
181
|
+
Both stages use classifier-free guidance (trained with 10% caption dropout) with a shared
|
|
182
|
+
`cond_scale`; 5 is the value selected by the CFG sweep and used for every headline
|
|
183
|
+
number. Sampling uses the EMA weights of the merged epoch-650 checkpoint. The presets
|
|
184
|
+
ship inside the package: [`rsdiff1.5`](src/rsdiff/configs/rsdiff1.5.yaml) (released) and
|
|
185
|
+
[`rsdiff1`](src/rsdiff/configs/rsdiff1.yaml) (the 723 M paper configuration; weights not
|
|
186
|
+
released).
|
|
187
|
+
|
|
188
|
+
## Limitations
|
|
189
|
+
|
|
190
|
+
- **Synthetic only.** Outputs are plausible-looking imagery, not observed or predicted
|
|
191
|
+
geography. Do not use them as evidence about a real location.
|
|
192
|
+
- **RSICD domain.** The model was trained on the 30 RSICD scene types and their short,
|
|
193
|
+
lowercase captions. Prompts unlike those captions have not been evaluated.
|
|
194
|
+
- **256 px.** The cascade stops at 256x256.
|
|
195
|
+
- **No Apple MPS.** float64 operations in imagen-pytorch 2.1.0 and torchmetrics FID are
|
|
196
|
+
not supported on MPS.
|
|
197
|
+
- **CPU sampling is slow.** 2,000 denoising steps per image took about an hour on an
|
|
198
|
+
Apple-silicon CPU; use a CUDA GPU for more than a handful of images.
|
|
199
|
+
- **Weights not commercially cleared.** RSICD's commercial terms are unresolved; see
|
|
200
|
+
[`docs/LICENSE_AUDIT.md`](docs/LICENSE_AUDIT.md).
|
|
201
|
+
|
|
202
|
+
## Training
|
|
203
|
+
|
|
204
|
+
Training still runs through the frozen reference implementation in [`ddpm/`](ddpm/):
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
bash ddpm/scripts/run_training_local.sh # stage 1: 128x128 base UNet
|
|
208
|
+
bash ddpm/scripts/run_sr.sh # stage 2: 256x256 SR UNet on the frozen base
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
[`docs/reproducibility.md`](docs/reproducibility.md) is the complete runbook: environment,
|
|
212
|
+
dataset preparation, training, milestone sweeps, sampling the 1,093-image bundle, and
|
|
213
|
+
scoring.
|
|
214
|
+
|
|
215
|
+
## Roadmap
|
|
216
|
+
|
|
217
|
+
1. **Reproduce with the package.** Regenerate the 1,093-image bundle with `rsdiff sample`
|
|
218
|
+
and match the published FID and CLIP-score; then delete `ddpm/`.
|
|
219
|
+
2. **Commercial relevance.** Test whether synthetic augmentation improves a downstream
|
|
220
|
+
model on an untouched real-data test set.
|
|
221
|
+
|
|
222
|
+
Details and acceptance criteria: [`docs/PLAN.md`](docs/PLAN.md).
|
|
223
|
+
|
|
224
|
+
## Citation
|
|
225
|
+
|
|
226
|
+
```bibtex
|
|
227
|
+
@article{sebaq2024rsdiff,
|
|
228
|
+
title = {RSDiff: remote sensing image generation from text using diffusion model},
|
|
229
|
+
author = {Sebaq, Ahmad and ElHelw, Mohamed},
|
|
230
|
+
journal = {Neural Computing and Applications},
|
|
231
|
+
volume = {36},
|
|
232
|
+
number = {36},
|
|
233
|
+
pages = {23103--23111},
|
|
234
|
+
year = {2024},
|
|
235
|
+
doi = {10.1007/s00521-024-10363-3}
|
|
236
|
+
}
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
## License
|
|
240
|
+
|
|
241
|
+
Code: Apache-2.0, see [`LICENSE`](LICENSE) and [`NOTICE`](NOTICE). The released weights
|
|
242
|
+
were trained on RSICD, whose commercial terms are unresolved; dataset, dependency, and
|
|
243
|
+
derived-weight rights are tracked in [`docs/LICENSE_AUDIT.md`](docs/LICENSE_AUDIT.md).
|
|
244
|
+
Built on [`lucidrains/imagen-pytorch`](https://github.com/lucidrains/imagen-pytorch);
|
|
245
|
+
developed at the Center for Informatics Science, Nile University.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
rsdiff/__init__.py,sha256=dFKNtf5sLNLYerXv1U5_rCqkv47ZAgEild_lbXT1JCE,72
|
|
2
|
+
rsdiff/cascade.py,sha256=jLtxb3uFCLBR3xlplgWXwnGL4LK6ufgkbfsPJxCprbY,6905
|
|
3
|
+
rsdiff/cli.py,sha256=GWWLmDCaTF7T7i346VV_jaIQZUSFmPX2wlPIud07AW4,5335
|
|
4
|
+
rsdiff/config.py,sha256=aTCDO3WBcYbHxEVeFyh1NvNKBT_ieegI3igBZnm7u3M,3272
|
|
5
|
+
rsdiff/configs/rsdiff1.5.yaml,sha256=iE88GumtvABPMv8T6B8JRM53GHwA7L7XyvKdDfWCWYM,1476
|
|
6
|
+
rsdiff/configs/rsdiff1.yaml,sha256=sQIz77iWh60CeYhttRJpneM0HEpPCbCXHQF91sfUUBk,1986
|
|
7
|
+
rsdiff/datasets/__init__.py,sha256=iWek0miCKlsqryty69wNtDB2H4HonY2I9G6HgYU9gYA,395
|
|
8
|
+
rsdiff/datasets/base.py,sha256=EnY4p3bBxHOCYOUT_zjzKbkiW3_rxa7hBRV71XYIWa4,1161
|
|
9
|
+
rsdiff/datasets/rsicd.py,sha256=bkucy-28f9F-FgaNucejywayd1oIoZTln9pFjk9OCw4,3053
|
|
10
|
+
rsdiff/eval/__init__.py,sha256=w_3fuw-_b0uinrfxfb8LR8TT1BoeTvlvxGJ5B1hUxbs,438
|
|
11
|
+
rsdiff/eval/clip.py,sha256=5rGKZy-O2EgvCNiypu7O9V5ECg--Wyl7oKGbYyo9feY,2969
|
|
12
|
+
rsdiff/eval/fid.py,sha256=Vfdc6NIgu6nyM0ZLF4EjG84ST23LMNmag-ebYS3Z8SQ,4401
|
|
13
|
+
rsdiff-0.1.0.dist-info/METADATA,sha256=GZBcFbvzqYdf_TyMeuiYPtF-m41fQNJlqv5YfipIM5s,11163
|
|
14
|
+
rsdiff-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
15
|
+
rsdiff-0.1.0.dist-info/entry_points.txt,sha256=d0In_rBx8UOk3W4TiPNvkl1XIXaGMC7d9SoOlgx0sig,43
|
|
16
|
+
rsdiff-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
17
|
+
rsdiff-0.1.0.dist-info/licenses/NOTICE,sha256=G_IlLEooLyCpILDADhExxse-DRNF6Nq3YbSzk2BGgwQ,1192
|
|
18
|
+
rsdiff-0.1.0.dist-info/RECORD,,
|