disrated 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disrated/__init__.py +2 -0
- disrated/audit_clip.py +76 -0
- disrated/audit_components.py +27 -0
- disrated/audit_dhash_residuals.py +65 -0
- disrated/audit_duplicate_labels.py +89 -0
- disrated/audit_effect.py +82 -0
- disrated/audit_hashes.py +106 -0
- disrated/data/__init__.py +0 -0
- disrated/data/splits.py +33 -0
- disrated/data/vhd11k.py +68 -0
- disrated/embed.py +74 -0
- disrated/evalrun.py +44 -0
- disrated/metrics/__init__.py +0 -0
- disrated/metrics/scoring.py +119 -0
- disrated/models/__init__.py +0 -0
- disrated/models/base.py +21 -0
- disrated/models/dummy.py +42 -0
- disrated/models/nudenet_mod.py +38 -0
- disrated/probe.py +170 -0
- disrated/rebuild_splits.py +132 -0
- disrated/risk_coverage.py +109 -0
- disrated/runner.py +106 -0
- disrated/schema.py +21 -0
- disrated/score.py +83 -0
- disrated/sweep_thresholds.py +101 -0
- disrated-0.1.0.dist-info/METADATA +16 -0
- disrated-0.1.0.dist-info/RECORD +29 -0
- disrated-0.1.0.dist-info/WHEEL +4 -0
- disrated-0.1.0.dist-info/entry_points.txt +3 -0
disrated/__init__.py
ADDED
disrated/audit_clip.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import random
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from .data.vhd11k import load_vhd11k
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def main() -> None:
|
|
13
|
+
ap = argparse.ArgumentParser(prog="python -m disrated.audit_clip")
|
|
14
|
+
ap.add_argument("--embeddings", type=Path, default=Path("embeddings/openai_clip-vit-base-patch16.npz"))
|
|
15
|
+
ap.add_argument("--dataset", type=Path, required=True)
|
|
16
|
+
ap.add_argument("--splits", type=Path, default=Path("splits.json"))
|
|
17
|
+
a = ap.parse_args()
|
|
18
|
+
|
|
19
|
+
samples, _ = load_vhd11k(a.dataset.expanduser())
|
|
20
|
+
by_id = {s.sample_id: s for s in samples}
|
|
21
|
+
|
|
22
|
+
z = np.load(a.embeddings, allow_pickle=False)
|
|
23
|
+
id_to_idx = {sid: idx for idx, sid in enumerate(z["ids"].tolist())}
|
|
24
|
+
feats = z["feats"] # shape (10000, 512), already L2 normalized
|
|
25
|
+
|
|
26
|
+
splits = json.loads(a.splits.read_text(encoding="utf-8"))
|
|
27
|
+
tr_idx = np.array([id_to_idx[sid] for sid in splits["train"]])
|
|
28
|
+
te_idx = np.array([id_to_idx[sid] for sid in splits["test"]])
|
|
29
|
+
|
|
30
|
+
# Test-to-Train max cosine similarities
|
|
31
|
+
X_tr = feats[tr_idx]
|
|
32
|
+
X_te = feats[te_idx]
|
|
33
|
+
sims = np.matmul(X_te, X_tr.T)
|
|
34
|
+
max_sims = np.max(sims, axis=1)
|
|
35
|
+
|
|
36
|
+
print("\n=== TEST -> TRAIN MAX COSINE SIMILARITY ===")
|
|
37
|
+
qs = [0, 25, 50, 75, 90, 95, 99, 100]
|
|
38
|
+
vals = np.percentile(max_sims, qs)
|
|
39
|
+
for q, v in zip(qs, vals):
|
|
40
|
+
print(f"p{q:<3}: {v:.4f}")
|
|
41
|
+
|
|
42
|
+
print("\nThreshold exceedances (Test N=1000):")
|
|
43
|
+
for t in [0.90, 0.95, 0.98]:
|
|
44
|
+
cnt = int(np.sum(max_sims >= t))
|
|
45
|
+
print(f"Cosine >= {t:.2f}: {cnt:>4} ({cnt / len(max_sims) * 100:.1f}%)")
|
|
46
|
+
|
|
47
|
+
# Same-prompt SDXL vs random SDXL pairs
|
|
48
|
+
sdxl_samples = [s for s in samples if s.source == "sdxl"]
|
|
49
|
+
groups: dict[str, list[str]] = {}
|
|
50
|
+
for s in sdxl_samples:
|
|
51
|
+
groups.setdefault(s.group, []).append(s.sample_id)
|
|
52
|
+
|
|
53
|
+
multi = [v for v in groups.values() if len(v) >= 2]
|
|
54
|
+
same_prompt_sims = []
|
|
55
|
+
for g in multi:
|
|
56
|
+
for i in range(len(g)):
|
|
57
|
+
for j in range(i + 1, len(g)):
|
|
58
|
+
same_prompt_sims.append(float(np.dot(feats[id_to_idx[g[i]]], feats[id_to_idx[g[j]]])))
|
|
59
|
+
|
|
60
|
+
rng = random.Random(0)
|
|
61
|
+
all_sdxl = [s.sample_id for s in sdxl_samples]
|
|
62
|
+
random_sdxl_sims = []
|
|
63
|
+
for _ in range(len(same_prompt_sims)):
|
|
64
|
+
i1, i2 = rng.sample(all_sdxl, 2)
|
|
65
|
+
if by_id[i1].group != by_id[i2].group:
|
|
66
|
+
random_sdxl_sims.append(float(np.dot(feats[id_to_idx[i1]], feats[id_to_idx[i2]])))
|
|
67
|
+
|
|
68
|
+
sp = np.array(same_prompt_sims)
|
|
69
|
+
rd = np.array(random_sdxl_sims)
|
|
70
|
+
print(f"\n=== SDXL PAIRWISE COSINE SIMILARITY (N={len(sp)}) ===")
|
|
71
|
+
print(f"Same prompt index: mean={sp.mean():.4f} | median={np.median(sp):.4f} | IQR={np.percentile(sp, 75) - np.percentile(sp, 25):.4f}")
|
|
72
|
+
print(f"Random prompt index: mean={rd.mean():.4f} | median={np.median(rd):.4f} | IQR={np.percentile(rd, 75) - np.percentile(rd, 25):.4f}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
if __name__ == "__main__":
|
|
76
|
+
main()
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from collections import Counter
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def main() -> None:
|
|
9
|
+
comp_map = json.loads(Path("embeddings/components.json").read_text(encoding="utf-8"))
|
|
10
|
+
counts = Counter(comp_map.values())
|
|
11
|
+
size_dist = Counter(counts.values())
|
|
12
|
+
|
|
13
|
+
print("\n=== COMPONENT SIZE DISTRIBUTION ===")
|
|
14
|
+
print(f"{'Component Size':<16} | {'Count':<8} | {'Total Images':<12}")
|
|
15
|
+
print("-" * 42)
|
|
16
|
+
for size in sorted(size_dist.keys()):
|
|
17
|
+
num_comps = size_dist[size]
|
|
18
|
+
print(f"{size:<16} | {num_comps:<8} | {num_comps * size:<12}")
|
|
19
|
+
|
|
20
|
+
largest_comp_id, max_size = counts.most_common(1)[0]
|
|
21
|
+
largest_members = [sid for sid, cid in comp_map.items() if cid == largest_comp_id]
|
|
22
|
+
print(f"\nLargest component: {largest_comp_id} (size = {max_size})")
|
|
23
|
+
print("Members (first 10):", largest_members[:10])
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
if __name__ == "__main__":
|
|
27
|
+
main()
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from .data.vhd11k import load_vhd11k
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def main() -> None:
|
|
12
|
+
ap = argparse.ArgumentParser(prog="python -m disrated.audit_dhash_residuals")
|
|
13
|
+
ap.add_argument("--dataset", type=Path, required=True)
|
|
14
|
+
ap.add_argument("--splits", type=Path, default=Path("splits.v2.json"))
|
|
15
|
+
ap.add_argument("--embeddings", type=Path, default=Path("embeddings/openai_clip-vit-base-patch16.npz"))
|
|
16
|
+
ap.add_argument("--hashes", type=Path, default=Path("embeddings/hashes.npz"))
|
|
17
|
+
a = ap.parse_args()
|
|
18
|
+
|
|
19
|
+
samples, _ = load_vhd11k(a.dataset.expanduser())
|
|
20
|
+
by_id = {s.sample_id: s for s in samples}
|
|
21
|
+
|
|
22
|
+
splits = json.loads(a.splits.read_text(encoding="utf-8"))
|
|
23
|
+
z_emb = np.load(a.embeddings, allow_pickle=False)
|
|
24
|
+
id_to_idx = {sid: idx for idx, sid in enumerate(z_emb["ids"].tolist())}
|
|
25
|
+
|
|
26
|
+
z_hash = np.load(a.hashes, allow_pickle=False)
|
|
27
|
+
dhash = z_hash["dhash"]
|
|
28
|
+
phash = z_hash["phash"]
|
|
29
|
+
feats = z_emb["feats"]
|
|
30
|
+
lut = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
|
|
31
|
+
|
|
32
|
+
tr_ids = splits["train"]
|
|
33
|
+
te_ids = splits["test"]
|
|
34
|
+
tr_idx = np.array([id_to_idx[sid] for sid in tr_ids])
|
|
35
|
+
te_idx = np.array([id_to_idx[sid] for sid in te_ids])
|
|
36
|
+
|
|
37
|
+
print("\n=== DHASH RESIDUAL PAIRS (train vs test, dist <= 4) ===")
|
|
38
|
+
print(f"{'Train ID':<35} {'Test ID':<35} {'dHash':<6} {'pHash':<6} {'Cosine':<7} {'Labels (Tr/Te)'}")
|
|
39
|
+
print("-" * 105)
|
|
40
|
+
|
|
41
|
+
for i, te_id in enumerate(te_ids):
|
|
42
|
+
t_idx = te_idx[i]
|
|
43
|
+
xor_d = np.bitwise_xor(dhash[t_idx:t_idx + 1], dhash[tr_idx])
|
|
44
|
+
d_dists = lut[xor_d].sum(axis=-1)
|
|
45
|
+
|
|
46
|
+
match_positions = np.where(d_dists <= 4)[0]
|
|
47
|
+
for tr_pos in match_positions:
|
|
48
|
+
tr_id = tr_ids[tr_pos]
|
|
49
|
+
d_val = int(d_dists[tr_pos])
|
|
50
|
+
|
|
51
|
+
# pHash distance
|
|
52
|
+
xor_p = np.bitwise_xor(phash[t_idx], phash[tr_idx[tr_pos]])
|
|
53
|
+
p_val = int(lut[xor_p].sum())
|
|
54
|
+
|
|
55
|
+
# Cosine similarity
|
|
56
|
+
cos_val = float(np.dot(feats[t_idx], feats[tr_idx[tr_pos]]))
|
|
57
|
+
|
|
58
|
+
l_tr = "Harm" if by_id[tr_id].harmful else "Safe"
|
|
59
|
+
l_te = "Harm" if by_id[te_id].harmful else "Safe"
|
|
60
|
+
|
|
61
|
+
print(f"{tr_id:<35} {te_id:<35} {d_val:<6} {p_val:<6} {cos_val:<7.4f} {l_tr}/{l_te}")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
if __name__ == "__main__":
|
|
65
|
+
main()
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from .data.vhd11k import load_vhd11k
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def main() -> None:
|
|
11
|
+
ap = argparse.ArgumentParser(prog="python -m disrated.audit_duplicate_labels")
|
|
12
|
+
ap.add_argument("--dataset", type=Path, required=True)
|
|
13
|
+
ap.add_argument("--embeddings", type=Path, default=Path("embeddings/openai_clip-vit-base-patch16.npz"))
|
|
14
|
+
ap.add_argument("--hashes", type=Path, default=Path("embeddings/hashes.npz"))
|
|
15
|
+
ap.add_argument("--cosine-threshold", type=float, default=0.95)
|
|
16
|
+
ap.add_argument("--phash-threshold", type=int, default=8)
|
|
17
|
+
a = ap.parse_args()
|
|
18
|
+
|
|
19
|
+
samples, _ = load_vhd11k(a.dataset.expanduser())
|
|
20
|
+
by_id = {s.sample_id: s for s in samples}
|
|
21
|
+
all_ids = [s.sample_id for s in samples]
|
|
22
|
+
|
|
23
|
+
z_emb = np.load(a.embeddings, allow_pickle=False)
|
|
24
|
+
feats = z_emb["feats"]
|
|
25
|
+
|
|
26
|
+
z_hash = np.load(a.hashes, allow_pickle=False)
|
|
27
|
+
phash = z_hash["phash"]
|
|
28
|
+
lut = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
|
|
29
|
+
|
|
30
|
+
pairs = set()
|
|
31
|
+
n = len(samples)
|
|
32
|
+
chunk = 1000
|
|
33
|
+
|
|
34
|
+
for i in range(0, n, chunk):
|
|
35
|
+
end_i = min(i + chunk, n)
|
|
36
|
+
# Cosine pairs
|
|
37
|
+
cos = np.matmul(feats[i:end_i], feats.T)
|
|
38
|
+
for r_off in range(end_i - i):
|
|
39
|
+
g_r = i + r_off
|
|
40
|
+
cos[r_off, :g_r + 1] = 0.0
|
|
41
|
+
r_c, c_c = np.where(cos >= a.cosine_threshold)
|
|
42
|
+
for r, c in zip(r_c, c_c):
|
|
43
|
+
pairs.add((i + r, c))
|
|
44
|
+
|
|
45
|
+
# pHash pairs
|
|
46
|
+
for r_off in range(end_i - i):
|
|
47
|
+
g_r = i + r_off
|
|
48
|
+
xor = np.bitwise_xor(phash[g_r:g_r + 1], phash[g_r + 1:])
|
|
49
|
+
dists = lut[xor].sum(axis=-1)
|
|
50
|
+
for col in np.where(dists <= a.phash_threshold)[0]:
|
|
51
|
+
pairs.add((g_r, g_r + 1 + col))
|
|
52
|
+
|
|
53
|
+
agreed = 0
|
|
54
|
+
disagreed = 0
|
|
55
|
+
by_type = {"concept_concept": [0, 0], "sdxl_sdxl": [0, 0], "cross": [0, 0]}
|
|
56
|
+
|
|
57
|
+
for i, j in pairs:
|
|
58
|
+
s1 = by_id[all_ids[i]]
|
|
59
|
+
s2 = by_id[all_ids[j]]
|
|
60
|
+
match = (s1.harmful == s2.harmful)
|
|
61
|
+
|
|
62
|
+
if match:
|
|
63
|
+
agreed += 1
|
|
64
|
+
else:
|
|
65
|
+
disagreed += 1
|
|
66
|
+
|
|
67
|
+
if s1.source == "concept" and s2.source == "concept":
|
|
68
|
+
key = "concept_concept"
|
|
69
|
+
elif s1.source == "sdxl" and s2.source == "sdxl":
|
|
70
|
+
key = "sdxl_sdxl"
|
|
71
|
+
else:
|
|
72
|
+
key = "cross"
|
|
73
|
+
|
|
74
|
+
by_type[key][0 if match else 1] += 1
|
|
75
|
+
|
|
76
|
+
total = len(pairs)
|
|
77
|
+
print("\n=== LABEL AGREEMENT AMONG NEAR-DUPLICATES ===")
|
|
78
|
+
print(f"Total unique duplicate pairs: {total}")
|
|
79
|
+
print(f"Same label: {agreed:>4} ({agreed / total * 100:.1f}%)")
|
|
80
|
+
print(f"Opposing label: {disagreed:>4} ({disagreed / total * 100:.1f}%)")
|
|
81
|
+
print("\nBreakdown by cohort (Agreed / Disagreed):")
|
|
82
|
+
for k, (agr, dgr) in by_type.items():
|
|
83
|
+
sub_tot = agr + dgr
|
|
84
|
+
pct = (agr / sub_tot * 100) if sub_tot else 0
|
|
85
|
+
print(f" {k:<16}: {agr:>4} agreed, {dgr:>4} disagreed ({pct:.1f}% agreement)")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
if __name__ == "__main__":
|
|
89
|
+
main()
|
disrated/audit_effect.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import numpy as np
|
|
7
|
+
from sklearn.metrics import roc_auc_score
|
|
8
|
+
|
|
9
|
+
from .data.vhd11k import load_vhd11k
|
|
10
|
+
from .metrics.scoring import read_predictions
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def bootstrap_ci(y_true: np.ndarray, y_score: np.ndarray, n_boot: int = 1000, seed: int = 0) -> tuple[float, float]:
|
|
14
|
+
rng = np.random.default_rng(seed)
|
|
15
|
+
n = len(y_true)
|
|
16
|
+
aucs = []
|
|
17
|
+
for _ in range(n_boot):
|
|
18
|
+
idx = rng.integers(0, n, size=n)
|
|
19
|
+
if len(np.unique(y_true[idx])) < 2:
|
|
20
|
+
continue
|
|
21
|
+
aucs.append(roc_auc_score(y_true[idx], y_score[idx]))
|
|
22
|
+
return float(np.percentile(aucs, 2.5)), float(np.percentile(aucs, 97.5))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def main() -> None:
|
|
26
|
+
ap = argparse.ArgumentParser(prog="python -m disrated.audit_effect")
|
|
27
|
+
ap.add_argument("--dataset", type=Path, required=True)
|
|
28
|
+
ap.add_argument("--predictions", type=Path, default=Path("results/clip-b16-test.jsonl"))
|
|
29
|
+
ap.add_argument("--embeddings", type=Path, default=Path("embeddings/openai_clip-vit-base-patch16.npz"))
|
|
30
|
+
ap.add_argument("--hashes", type=Path, default=Path("embeddings/hashes.npz"))
|
|
31
|
+
ap.add_argument("--splits", type=Path, default=Path("splits.json"))
|
|
32
|
+
ap.add_argument("--cosine-threshold", type=float, required=True)
|
|
33
|
+
a = ap.parse_args()
|
|
34
|
+
|
|
35
|
+
samples, _ = load_vhd11k(a.dataset.expanduser())
|
|
36
|
+
by_id = {s.sample_id: s for s in samples}
|
|
37
|
+
splits = json.loads(a.splits.read_text(encoding="utf-8"))
|
|
38
|
+
preds = read_predictions(a.predictions)
|
|
39
|
+
|
|
40
|
+
z_emb = np.load(a.embeddings, allow_pickle=False)
|
|
41
|
+
id_to_idx = {sid: idx for idx, sid in enumerate(z_emb["ids"].tolist())}
|
|
42
|
+
tr_idx = np.array([id_to_idx[sid] for sid in splits["train"]])
|
|
43
|
+
te_ids = splits["test"]
|
|
44
|
+
te_idx = np.array([id_to_idx[sid] for sid in te_ids])
|
|
45
|
+
|
|
46
|
+
# Cosine check
|
|
47
|
+
X_tr = z_emb["feats"][tr_idx]
|
|
48
|
+
X_te = z_emb["feats"][te_idx]
|
|
49
|
+
max_cos = np.max(np.matmul(X_te, X_tr.T), axis=1)
|
|
50
|
+
cos_leaked = set(np.where(max_cos >= a.cosine_threshold)[0])
|
|
51
|
+
|
|
52
|
+
# pHash check
|
|
53
|
+
z_hash = np.load(a.hashes, allow_pickle=False)
|
|
54
|
+
lut = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
|
|
55
|
+
ph_tr = z_hash["phash"][tr_idx]
|
|
56
|
+
ph_te = z_hash["phash"][te_idx]
|
|
57
|
+
|
|
58
|
+
ph_leaked = set()
|
|
59
|
+
for i in range(len(te_idx)):
|
|
60
|
+
xor = np.bitwise_xor(ph_te[i:i + 1], ph_tr)
|
|
61
|
+
if np.any(lut[xor].sum(axis=-1) <= 8):
|
|
62
|
+
ph_leaked.add(i)
|
|
63
|
+
|
|
64
|
+
leaked_indices = cos_leaked | ph_leaked
|
|
65
|
+
mask_leaked = np.array([i in leaked_indices for i in range(len(te_ids))])
|
|
66
|
+
|
|
67
|
+
y_true = np.array([by_id[sid].harmful for sid in te_ids])
|
|
68
|
+
y_score = np.array([preds[sid].score for sid in te_ids])
|
|
69
|
+
|
|
70
|
+
print("\n=== EFFECT SIZE ON TEST SPLIT ===")
|
|
71
|
+
for label, mask in [("With Train Duplicate", mask_leaked), ("Without Train Duplicate", ~mask_leaked)]:
|
|
72
|
+
n_group = int(np.sum(mask))
|
|
73
|
+
if n_group == 0 or len(np.unique(y_true[mask])) < 2:
|
|
74
|
+
print(f"{label:<28}: n={n_group:>4} | undefined (insufficient class variation)")
|
|
75
|
+
continue
|
|
76
|
+
auc_val = roc_auc_score(y_true[mask], y_score[mask])
|
|
77
|
+
ci_lo, ci_hi = bootstrap_ci(y_true[mask], y_score[mask])
|
|
78
|
+
print(f"{label:<28}: n={n_group:>4} | AUC={auc_val:.4f} [95% CI: {ci_lo:.4f}, {ci_hi:.4f}]")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
if __name__ == "__main__":
|
|
82
|
+
main()
|
disrated/audit_hashes.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import imagehash
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from .data.vhd11k import load_vhd11k
|
|
10
|
+
from .runner import load_clean
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def compute_hashes(samples, cache_path: Path) -> dict[str, np.ndarray]:
|
|
14
|
+
if cache_path.exists():
|
|
15
|
+
z = np.load(cache_path, allow_pickle=False)
|
|
16
|
+
return {"ids": z["ids"], "phash": z["phash"], "dhash": z["dhash"]}
|
|
17
|
+
|
|
18
|
+
ids, ph_bits, dh_bits = [], [], []
|
|
19
|
+
for i, s in enumerate(samples):
|
|
20
|
+
img = load_clean(s.path)
|
|
21
|
+
ids.append(s.sample_id)
|
|
22
|
+
ph_bits.append(imagehash.phash(img).hash.flatten())
|
|
23
|
+
dh_bits.append(imagehash.dhash(img).hash.flatten())
|
|
24
|
+
if (i + 1) % 1000 == 0:
|
|
25
|
+
print(f"hashed {i + 1}/{len(samples)}", flush=True)
|
|
26
|
+
|
|
27
|
+
ids_arr = np.array(ids)
|
|
28
|
+
phash_packed = np.packbits(np.array(ph_bits, dtype=bool), axis=1)
|
|
29
|
+
dhash_packed = np.packbits(np.array(dh_bits, dtype=bool), axis=1)
|
|
30
|
+
|
|
31
|
+
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
|
32
|
+
np.savez_compressed(cache_path, ids=ids_arr, phash=phash_packed, dhash=dhash_packed)
|
|
33
|
+
return {"ids": ids_arr, "phash": phash_packed, "dhash": dhash_packed}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def hamming_chunked(a: np.ndarray, b: np.ndarray, chunk_size: int = 500) -> np.ndarray:
|
|
37
|
+
lut = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
|
|
38
|
+
dists = np.zeros((a.shape[0], b.shape[0]), dtype=np.uint8)
|
|
39
|
+
for start in range(0, a.shape[0], chunk_size):
|
|
40
|
+
end = min(start + chunk_size, a.shape[0])
|
|
41
|
+
xor = np.bitwise_xor(a[start:end, None, :], b[None, :, :])
|
|
42
|
+
dists[start:end] = lut[xor].sum(axis=-1)
|
|
43
|
+
return dists
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def audit_hamming(samples, hashes: dict[str, np.ndarray], splits_path: Path) -> dict:
|
|
47
|
+
splits = json.loads(splits_path.read_text(encoding="utf-8"))
|
|
48
|
+
id_to_idx = {sid: idx for idx, sid in enumerate(hashes["ids"].tolist())}
|
|
49
|
+
by_id = {s.sample_id: s for s in samples}
|
|
50
|
+
|
|
51
|
+
split_pairs = [
|
|
52
|
+
("train", "val", splits["train"], splits["val"]),
|
|
53
|
+
("train", "test", splits["train"], splits["test"]),
|
|
54
|
+
("val", "test", splits["val"], splits["test"]),
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
findings = {}
|
|
58
|
+
for hname in ["phash", "dhash"]:
|
|
59
|
+
print(f"\n=== {hname.upper()} CROSS-SPLIT HAMMING AUDIT ===")
|
|
60
|
+
data = hashes[hname]
|
|
61
|
+
for s1_name, s2_name, ids1, ids2 in split_pairs:
|
|
62
|
+
idx1 = np.array([id_to_idx[sid] for sid in ids1])
|
|
63
|
+
idx2 = np.array([id_to_idx[sid] for sid in ids2])
|
|
64
|
+
dists = hamming_chunked(data[idx1], data[idx2])
|
|
65
|
+
|
|
66
|
+
pair_key = f"{s1_name}_{s2_name}"
|
|
67
|
+
findings.setdefault(hname, {})[pair_key] = {}
|
|
68
|
+
|
|
69
|
+
print(f"\n--- {s1_name} ({len(ids1)}) vs {s2_name} ({len(ids2)}) ---")
|
|
70
|
+
for t in [0, 4, 8]:
|
|
71
|
+
r, c = np.where(dists <= t)
|
|
72
|
+
breakdown = {"concept_concept": 0, "sdxl_sdxl": 0, "cross_source": 0}
|
|
73
|
+
for i_pos, j_pos in zip(r, c):
|
|
74
|
+
s1 = by_id[ids1[i_pos]]
|
|
75
|
+
s2 = by_id[ids2[j_pos]]
|
|
76
|
+
if s1.source == "concept" and s2.source == "concept":
|
|
77
|
+
breakdown["concept_concept"] += 1
|
|
78
|
+
elif s1.source == "sdxl" and s2.source == "sdxl":
|
|
79
|
+
breakdown["sdxl_sdxl"] += 1
|
|
80
|
+
else:
|
|
81
|
+
breakdown["cross_source"] += 1
|
|
82
|
+
|
|
83
|
+
findings[hname][pair_key][t] = {"total": len(r), **breakdown}
|
|
84
|
+
print(
|
|
85
|
+
f"Dist <= {t:>2}: total={len(r):>5} | "
|
|
86
|
+
f"concept-concept={breakdown['concept_concept']:>4} | "
|
|
87
|
+
f"sdxl-sdxl={breakdown['sdxl_sdxl']:>4} | "
|
|
88
|
+
f"cross-source={breakdown['cross_source']:>4}"
|
|
89
|
+
)
|
|
90
|
+
return findings
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def main() -> None:
|
|
94
|
+
ap = argparse.ArgumentParser(prog="python -m disrated.audit_hashes")
|
|
95
|
+
ap.add_argument("--dataset", type=Path, required=True)
|
|
96
|
+
ap.add_argument("--splits", type=Path, default=Path("splits.json"))
|
|
97
|
+
ap.add_argument("--cache", type=Path, default=Path("embeddings/hashes.npz"))
|
|
98
|
+
a = ap.parse_args()
|
|
99
|
+
|
|
100
|
+
samples, _ = load_vhd11k(a.dataset.expanduser())
|
|
101
|
+
h = compute_hashes(samples, a.cache)
|
|
102
|
+
audit_hamming(samples, h, a.splits)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
if __name__ == "__main__":
|
|
106
|
+
main()
|
|
File without changes
|
disrated/data/splits.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
from sklearn.model_selection import StratifiedGroupKFold
|
|
8
|
+
|
|
9
|
+
from .vhd11k import Sample
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def make_splits(samples: list[Sample], seed: int = 0, path: Path | None = None) -> dict[str, list[str]]:
|
|
13
|
+
"""80/10/10 train/val/test, stratified on source x label, grouped so duplicates never straddle splits.
|
|
14
|
+
If `path` exists it is loaded as-is. Splits are written once and never regenerated."""
|
|
15
|
+
if path is not None and path.exists():
|
|
16
|
+
return json.loads(path.read_text())
|
|
17
|
+
|
|
18
|
+
y = [f"{s.source}|{int(s.harmful)}" for s in samples]
|
|
19
|
+
groups = [s.group for s in samples]
|
|
20
|
+
cv = StratifiedGroupKFold(n_splits=10, shuffle=True, random_state=seed)
|
|
21
|
+
folds = [idx for _, idx in cv.split(np.zeros(len(samples)), y, groups)]
|
|
22
|
+
|
|
23
|
+
def ids(idxs) -> list[str]:
|
|
24
|
+
return sorted(samples[i].sample_id for i in idxs)
|
|
25
|
+
|
|
26
|
+
out = {
|
|
27
|
+
"test": ids(folds[0]),
|
|
28
|
+
"val": ids(folds[1]),
|
|
29
|
+
"train": ids(i for f in folds[2:] for i in f),
|
|
30
|
+
}
|
|
31
|
+
if path is not None:
|
|
32
|
+
path.write_text(json.dumps(out, indent=1))
|
|
33
|
+
return out
|
disrated/data/vhd11k.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict
|
|
9
|
+
|
|
10
|
+
csv.field_size_limit(2**31 - 1) # debate columns are long
|
|
11
|
+
|
|
12
|
+
ANN_FILE = "harmful_image_10000_ann.csv"
|
|
13
|
+
IMG_DIR = "all_10000_evenHarmfulUnharmful"
|
|
14
|
+
|
|
15
|
+
_SDXL = re.compile(r"^sdxl_extend_by_llama3_(harmful|unharmful)_(\d+)_seed(\d+)\.png$")
|
|
16
|
+
_CONCEPT = re.compile(r"^(.+)_(\d+)_(\d+)\.\w+$")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Sample(BaseModel):
|
|
20
|
+
model_config = ConfigDict(frozen=True)
|
|
21
|
+
|
|
22
|
+
sample_id: str # the CSV imagePath; unique
|
|
23
|
+
path: Path
|
|
24
|
+
harmful: bool # ground truth: the CSV `decision`
|
|
25
|
+
harm_type: str # free-text VLM reason; error analysis only
|
|
26
|
+
source: Literal["sdxl", "concept"]
|
|
27
|
+
concept: str | None = None # one of ~45 tags, concept-named half only
|
|
28
|
+
intent: bool | None = None # what the filename claims (sdxl only). NEVER show to a model.
|
|
29
|
+
group: str # split unit, so near-duplicates stay together
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def parse_name(name: str) -> dict:
|
|
33
|
+
if m := _SDXL.match(name):
|
|
34
|
+
intent = m.group(1) == "harmful"
|
|
35
|
+
# same prompt index across seeds -> same group
|
|
36
|
+
return dict(source="sdxl", intent=intent, group=f"sdxl:{m.group(1)}:{m.group(2)}")
|
|
37
|
+
if m := _CONCEPT.match(name):
|
|
38
|
+
# no known duplicate structure here, so each image is its own group
|
|
39
|
+
return dict(source="concept", concept=m.group(1), group=name)
|
|
40
|
+
raise ValueError(f"unrecognised filename: {name}")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def load_vhd11k(root: Path) -> tuple[list[Sample], list[tuple[str, str]]]:
|
|
44
|
+
"""Returns (samples, problems). Bad rows are logged and skipped, never fatal."""
|
|
45
|
+
root = Path(root)
|
|
46
|
+
samples: list[Sample] = []
|
|
47
|
+
problems: list[tuple[str, str]] = []
|
|
48
|
+
with (root / ANN_FILE).open(newline="", encoding="utf-8") as f:
|
|
49
|
+
for row in csv.DictReader(f):
|
|
50
|
+
name = row["imagePath"]
|
|
51
|
+
path = root / IMG_DIR / name
|
|
52
|
+
try:
|
|
53
|
+
if row["decision"] not in ("yes", "no"):
|
|
54
|
+
raise ValueError(f"bad decision {row['decision']!r}")
|
|
55
|
+
if not path.is_file() or path.stat().st_size == 0:
|
|
56
|
+
raise ValueError("missing or empty file")
|
|
57
|
+
samples.append(
|
|
58
|
+
Sample(
|
|
59
|
+
sample_id=name,
|
|
60
|
+
path=path,
|
|
61
|
+
harmful=row["decision"] == "yes",
|
|
62
|
+
harm_type=row["harmfulType"],
|
|
63
|
+
**parse_name(name),
|
|
64
|
+
)
|
|
65
|
+
)
|
|
66
|
+
except ValueError as e:
|
|
67
|
+
problems.append((name, str(e)))
|
|
68
|
+
return samples, problems
|
disrated/embed.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import time
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import torch
|
|
9
|
+
from transformers import CLIPModel, CLIPProcessor
|
|
10
|
+
|
|
11
|
+
from .data.vhd11k import load_vhd11k
|
|
12
|
+
from .runner import load_clean
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def device() -> str:
|
|
16
|
+
if torch.cuda.is_available():
|
|
17
|
+
return "cuda"
|
|
18
|
+
if torch.backends.mps.is_available():
|
|
19
|
+
return "mps"
|
|
20
|
+
return "cpu"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def sync(dev: str) -> None:
|
|
24
|
+
if dev == "cuda":
|
|
25
|
+
torch.cuda.synchronize()
|
|
26
|
+
elif dev == "mps":
|
|
27
|
+
torch.mps.synchronize()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def main() -> None:
|
|
31
|
+
ap = argparse.ArgumentParser(prog="python -m disrated.embed")
|
|
32
|
+
ap.add_argument("--model", default="openai/clip-vit-base-patch16")
|
|
33
|
+
ap.add_argument("--dataset", type=Path, required=True)
|
|
34
|
+
ap.add_argument("--batch-size", type=int, default=32)
|
|
35
|
+
ap.add_argument("--limit", type=int)
|
|
36
|
+
ap.add_argument("--output", type=Path)
|
|
37
|
+
a = ap.parse_args()
|
|
38
|
+
|
|
39
|
+
samples, _ = load_vhd11k(a.dataset.expanduser())
|
|
40
|
+
if a.limit:
|
|
41
|
+
samples = samples[: a.limit]
|
|
42
|
+
dev = device()
|
|
43
|
+
print("device:", dev, "| images:", len(samples))
|
|
44
|
+
model = CLIPModel.from_pretrained(a.model).to(dev).eval()
|
|
45
|
+
proc = CLIPProcessor.from_pretrained(a.model)
|
|
46
|
+
|
|
47
|
+
feats, ms = [], []
|
|
48
|
+
with torch.no_grad():
|
|
49
|
+
for i in range(0, len(samples), a.batch_size):
|
|
50
|
+
batch = samples[i : i + a.batch_size]
|
|
51
|
+
imgs = [load_clean(s.path) for s in batch] # pixels only, as always
|
|
52
|
+
px = proc(images=imgs, return_tensors="pt")["pixel_values"].to(dev)
|
|
53
|
+
sync(dev)
|
|
54
|
+
t0 = time.perf_counter()
|
|
55
|
+
out = model.get_image_features(pixel_values=px)
|
|
56
|
+
f = out if torch.is_tensor(out) else out.pooler_output
|
|
57
|
+
f = torch.nn.functional.normalize(f.float(), dim=-1)
|
|
58
|
+
sync(dev)
|
|
59
|
+
dt = (time.perf_counter() - t0) * 1000 / len(batch) # model time only
|
|
60
|
+
feats.append(f.cpu().numpy())
|
|
61
|
+
ms += [dt] * len(batch)
|
|
62
|
+
if (i // a.batch_size) % 20 == 0:
|
|
63
|
+
print(f"{i}/{len(samples)}", flush=True)
|
|
64
|
+
|
|
65
|
+
name = a.model.replace("/", "_") + (f"-n{a.limit}" if a.limit else "")
|
|
66
|
+
out_path = a.output or Path("embeddings") / f"{name}.npz"
|
|
67
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
68
|
+
np.savez(out_path, ids=np.array([s.sample_id for s in samples]),
|
|
69
|
+
feats=np.concatenate(feats), ms=np.array(ms, dtype=np.float32))
|
|
70
|
+
print("wrote", out_path)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
if __name__ == "__main__":
|
|
74
|
+
main()
|
disrated/evalrun.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from .data.vhd11k import load_vhd11k
|
|
8
|
+
from .runner import run
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def build(name: str):
|
|
12
|
+
if name == "random":
|
|
13
|
+
from .models.dummy import Random
|
|
14
|
+
return Random(0)
|
|
15
|
+
if name.startswith("constant-"):
|
|
16
|
+
from .models.dummy import Constant
|
|
17
|
+
return Constant(float(name.split("-")[1]))
|
|
18
|
+
if name == "nudenet":
|
|
19
|
+
from .models.nudenet_mod import NudeNet
|
|
20
|
+
return NudeNet()
|
|
21
|
+
raise SystemExit(f"unknown model: {name}")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main() -> None:
|
|
25
|
+
ap = argparse.ArgumentParser(prog="python -m disrated.evalrun")
|
|
26
|
+
ap.add_argument("--model", required=True)
|
|
27
|
+
ap.add_argument("--dataset", type=Path, required=True)
|
|
28
|
+
ap.add_argument("--split", default="test", choices=["train", "val", "test", "all"])
|
|
29
|
+
ap.add_argument("--batch-size", type=int, default=16)
|
|
30
|
+
ap.add_argument("--limit", type=int)
|
|
31
|
+
ap.add_argument("--output", type=Path)
|
|
32
|
+
a = ap.parse_args()
|
|
33
|
+
|
|
34
|
+
samples, _ = load_vhd11k(a.dataset.expanduser())
|
|
35
|
+
if a.split != "all":
|
|
36
|
+
ids = set(json.loads(a.splits.read_text())[a.split])
|
|
37
|
+
samples = [s for s in samples if s.sample_id in ids]
|
|
38
|
+
out = a.output or Path(f"results/{a.model}-{a.split}.jsonl")
|
|
39
|
+
print(run(build(a.model), samples, out, a.batch_size, a.limit))
|
|
40
|
+
print("wrote", out)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
main()
|
|
File without changes
|