disrated 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,16 @@
1
+ Metadata-Version: 2.3
2
+ Name: disrated
3
+ Version: 0.1.0
4
+ Summary: Add your description here
5
+ Requires-Dist: imagehash>=4.3.2
6
+ Requires-Dist: nudenet>=3.4.2
7
+ Requires-Dist: numpy>=2.5.4
8
+ Requires-Dist: pillow>=12.3.0
9
+ Requires-Dist: pydantic>=2.13.5
10
+ Requires-Dist: scikit-learn>=1.6.1
11
+ Requires-Dist: torch>=2.14.1
12
+ Requires-Dist: torchvision>=0.29.1
13
+ Requires-Dist: transformers>=5.19.0
14
+ Requires-Python: >=3.12
15
+ Description-Content-Type: text/markdown
16
+
File without changes
@@ -0,0 +1,27 @@
1
+ [project]
2
+ name = "disrated"
3
+ version = "0.1.0"
4
+ description = "Add your description here"
5
+ readme = "README.md"
6
+ requires-python = ">=3.12"
7
+ dependencies = [
8
+ "imagehash>=4.3.2",
9
+ "nudenet>=3.4.2",
10
+ "numpy>=2.5.4",
11
+ "pillow>=12.3.0",
12
+ "pydantic>=2.13.5",
13
+ "scikit-learn>=1.6.1",
14
+ "torch>=2.14.1",
15
+ "torchvision>=0.29.1",
16
+ "transformers>=5.19.0",
17
+ ]
18
+
19
+ [project.scripts]
20
+ disrated = "disrated:main"
21
+
22
+ [build-system]
23
+ requires = ["uv_build>=0.13.0,<0.14.0"]
24
+ build-backend = "uv_build"
25
+
26
+ [dependency-groups]
27
+ dev = ["pytest>=8.4.2"]
@@ -0,0 +1,29 @@
1
+ [project]
2
+ name = "disrated"
3
+ version = "0.1.0"
4
+ description = "Add your description here"
5
+ readme = "README.md"
6
+ requires-python = ">=3.12"
7
+ dependencies = [
8
+ "imagehash>=4.3.2",
9
+ "nudenet>=3.4.2",
10
+ "numpy>=2.5.4",
11
+ "pillow>=12.3.0",
12
+ "pydantic>=2.13.5",
13
+ "scikit-learn>=1.6.1",
14
+ "torch>=2.14.1",
15
+ "torchvision>=0.29.1",
16
+ "transformers>=5.19.0",
17
+ ]
18
+
19
+ [project.scripts]
20
+ disrated = "disrated:main"
21
+
22
+ [build-system]
23
+ requires = ["uv_build>=0.13.0,<0.14.0"]
24
+ build-backend = "uv_build"
25
+
26
+ [dependency-groups]
27
+ dev = [
28
+ "pytest>=8.4.2",
29
+ ]
@@ -0,0 +1,2 @@
1
+ def main() -> None:
2
+ print("Hello from disrated!")
@@ -0,0 +1,76 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import json
5
+ import random
6
+ from pathlib import Path
7
+ import numpy as np
8
+
9
+ from .data.vhd11k import load_vhd11k
10
+
11
+
12
+ def main() -> None:
13
+ ap = argparse.ArgumentParser(prog="python -m disrated.audit_clip")
14
+ ap.add_argument("--embeddings", type=Path, default=Path("embeddings/openai_clip-vit-base-patch16.npz"))
15
+ ap.add_argument("--dataset", type=Path, required=True)
16
+ ap.add_argument("--splits", type=Path, default=Path("splits.json"))
17
+ a = ap.parse_args()
18
+
19
+ samples, _ = load_vhd11k(a.dataset.expanduser())
20
+ by_id = {s.sample_id: s for s in samples}
21
+
22
+ z = np.load(a.embeddings, allow_pickle=False)
23
+ id_to_idx = {sid: idx for idx, sid in enumerate(z["ids"].tolist())}
24
+ feats = z["feats"] # shape (10000, 512), already L2 normalized
25
+
26
+ splits = json.loads(a.splits.read_text(encoding="utf-8"))
27
+ tr_idx = np.array([id_to_idx[sid] for sid in splits["train"]])
28
+ te_idx = np.array([id_to_idx[sid] for sid in splits["test"]])
29
+
30
+ # Test-to-Train max cosine similarities
31
+ X_tr = feats[tr_idx]
32
+ X_te = feats[te_idx]
33
+ sims = np.matmul(X_te, X_tr.T)
34
+ max_sims = np.max(sims, axis=1)
35
+
36
+ print("\n=== TEST -> TRAIN MAX COSINE SIMILARITY ===")
37
+ qs = [0, 25, 50, 75, 90, 95, 99, 100]
38
+ vals = np.percentile(max_sims, qs)
39
+ for q, v in zip(qs, vals):
40
+ print(f"p{q:<3}: {v:.4f}")
41
+
42
+ print("\nThreshold exceedances (Test N=1000):")
43
+ for t in [0.90, 0.95, 0.98]:
44
+ cnt = int(np.sum(max_sims >= t))
45
+ print(f"Cosine >= {t:.2f}: {cnt:>4} ({cnt / len(max_sims) * 100:.1f}%)")
46
+
47
+ # Same-prompt SDXL vs random SDXL pairs
48
+ sdxl_samples = [s for s in samples if s.source == "sdxl"]
49
+ groups: dict[str, list[str]] = {}
50
+ for s in sdxl_samples:
51
+ groups.setdefault(s.group, []).append(s.sample_id)
52
+
53
+ multi = [v for v in groups.values() if len(v) >= 2]
54
+ same_prompt_sims = []
55
+ for g in multi:
56
+ for i in range(len(g)):
57
+ for j in range(i + 1, len(g)):
58
+ same_prompt_sims.append(float(np.dot(feats[id_to_idx[g[i]]], feats[id_to_idx[g[j]]])))
59
+
60
+ rng = random.Random(0)
61
+ all_sdxl = [s.sample_id for s in sdxl_samples]
62
+ random_sdxl_sims = []
63
+ for _ in range(len(same_prompt_sims)):
64
+ i1, i2 = rng.sample(all_sdxl, 2)
65
+ if by_id[i1].group != by_id[i2].group:
66
+ random_sdxl_sims.append(float(np.dot(feats[id_to_idx[i1]], feats[id_to_idx[i2]])))
67
+
68
+ sp = np.array(same_prompt_sims)
69
+ rd = np.array(random_sdxl_sims)
70
+ print(f"\n=== SDXL PAIRWISE COSINE SIMILARITY (N={len(sp)}) ===")
71
+ print(f"Same prompt index: mean={sp.mean():.4f} | median={np.median(sp):.4f} | IQR={np.percentile(sp, 75) - np.percentile(sp, 25):.4f}")
72
+ print(f"Random prompt index: mean={rd.mean():.4f} | median={np.median(rd):.4f} | IQR={np.percentile(rd, 75) - np.percentile(rd, 25):.4f}")
73
+
74
+
75
+ if __name__ == "__main__":
76
+ main()
@@ -0,0 +1,27 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from collections import Counter
5
+ from pathlib import Path
6
+
7
+
8
+ def main() -> None:
9
+ comp_map = json.loads(Path("embeddings/components.json").read_text(encoding="utf-8"))
10
+ counts = Counter(comp_map.values())
11
+ size_dist = Counter(counts.values())
12
+
13
+ print("\n=== COMPONENT SIZE DISTRIBUTION ===")
14
+ print(f"{'Component Size':<16} | {'Count':<8} | {'Total Images':<12}")
15
+ print("-" * 42)
16
+ for size in sorted(size_dist.keys()):
17
+ num_comps = size_dist[size]
18
+ print(f"{size:<16} | {num_comps:<8} | {num_comps * size:<12}")
19
+
20
+ largest_comp_id, max_size = counts.most_common(1)[0]
21
+ largest_members = [sid for sid, cid in comp_map.items() if cid == largest_comp_id]
22
+ print(f"\nLargest component: {largest_comp_id} (size = {max_size})")
23
+ print("Members (first 10):", largest_members[:10])
24
+
25
+
26
+ if __name__ == "__main__":
27
+ main()
@@ -0,0 +1,65 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import json
5
+ from pathlib import Path
6
+ import numpy as np
7
+
8
+ from .data.vhd11k import load_vhd11k
9
+
10
+
11
+ def main() -> None:
12
+ ap = argparse.ArgumentParser(prog="python -m disrated.audit_dhash_residuals")
13
+ ap.add_argument("--dataset", type=Path, required=True)
14
+ ap.add_argument("--splits", type=Path, default=Path("splits.v2.json"))
15
+ ap.add_argument("--embeddings", type=Path, default=Path("embeddings/openai_clip-vit-base-patch16.npz"))
16
+ ap.add_argument("--hashes", type=Path, default=Path("embeddings/hashes.npz"))
17
+ a = ap.parse_args()
18
+
19
+ samples, _ = load_vhd11k(a.dataset.expanduser())
20
+ by_id = {s.sample_id: s for s in samples}
21
+
22
+ splits = json.loads(a.splits.read_text(encoding="utf-8"))
23
+ z_emb = np.load(a.embeddings, allow_pickle=False)
24
+ id_to_idx = {sid: idx for idx, sid in enumerate(z_emb["ids"].tolist())}
25
+
26
+ z_hash = np.load(a.hashes, allow_pickle=False)
27
+ dhash = z_hash["dhash"]
28
+ phash = z_hash["phash"]
29
+ feats = z_emb["feats"]
30
+ lut = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
31
+
32
+ tr_ids = splits["train"]
33
+ te_ids = splits["test"]
34
+ tr_idx = np.array([id_to_idx[sid] for sid in tr_ids])
35
+ te_idx = np.array([id_to_idx[sid] for sid in te_ids])
36
+
37
+ print("\n=== DHASH RESIDUAL PAIRS (train vs test, dist <= 4) ===")
38
+ print(f"{'Train ID':<35} {'Test ID':<35} {'dHash':<6} {'pHash':<6} {'Cosine':<7} {'Labels (Tr/Te)'}")
39
+ print("-" * 105)
40
+
41
+ for i, te_id in enumerate(te_ids):
42
+ t_idx = te_idx[i]
43
+ xor_d = np.bitwise_xor(dhash[t_idx:t_idx + 1], dhash[tr_idx])
44
+ d_dists = lut[xor_d].sum(axis=-1)
45
+
46
+ match_positions = np.where(d_dists <= 4)[0]
47
+ for tr_pos in match_positions:
48
+ tr_id = tr_ids[tr_pos]
49
+ d_val = int(d_dists[tr_pos])
50
+
51
+ # pHash distance
52
+ xor_p = np.bitwise_xor(phash[t_idx], phash[tr_idx[tr_pos]])
53
+ p_val = int(lut[xor_p].sum())
54
+
55
+ # Cosine similarity
56
+ cos_val = float(np.dot(feats[t_idx], feats[tr_idx[tr_pos]]))
57
+
58
+ l_tr = "Harm" if by_id[tr_id].harmful else "Safe"
59
+ l_te = "Harm" if by_id[te_id].harmful else "Safe"
60
+
61
+ print(f"{tr_id:<35} {te_id:<35} {d_val:<6} {p_val:<6} {cos_val:<7.4f} {l_tr}/{l_te}")
62
+
63
+
64
+ if __name__ == "__main__":
65
+ main()
@@ -0,0 +1,89 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ from pathlib import Path
5
+ import numpy as np
6
+
7
+ from .data.vhd11k import load_vhd11k
8
+
9
+
10
+ def main() -> None:
11
+ ap = argparse.ArgumentParser(prog="python -m disrated.audit_duplicate_labels")
12
+ ap.add_argument("--dataset", type=Path, required=True)
13
+ ap.add_argument("--embeddings", type=Path, default=Path("embeddings/openai_clip-vit-base-patch16.npz"))
14
+ ap.add_argument("--hashes", type=Path, default=Path("embeddings/hashes.npz"))
15
+ ap.add_argument("--cosine-threshold", type=float, default=0.95)
16
+ ap.add_argument("--phash-threshold", type=int, default=8)
17
+ a = ap.parse_args()
18
+
19
+ samples, _ = load_vhd11k(a.dataset.expanduser())
20
+ by_id = {s.sample_id: s for s in samples}
21
+ all_ids = [s.sample_id for s in samples]
22
+
23
+ z_emb = np.load(a.embeddings, allow_pickle=False)
24
+ feats = z_emb["feats"]
25
+
26
+ z_hash = np.load(a.hashes, allow_pickle=False)
27
+ phash = z_hash["phash"]
28
+ lut = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
29
+
30
+ pairs = set()
31
+ n = len(samples)
32
+ chunk = 1000
33
+
34
+ for i in range(0, n, chunk):
35
+ end_i = min(i + chunk, n)
36
+ # Cosine pairs
37
+ cos = np.matmul(feats[i:end_i], feats.T)
38
+ for r_off in range(end_i - i):
39
+ g_r = i + r_off
40
+ cos[r_off, :g_r + 1] = 0.0
41
+ r_c, c_c = np.where(cos >= a.cosine_threshold)
42
+ for r, c in zip(r_c, c_c):
43
+ pairs.add((i + r, c))
44
+
45
+ # pHash pairs
46
+ for r_off in range(end_i - i):
47
+ g_r = i + r_off
48
+ xor = np.bitwise_xor(phash[g_r:g_r + 1], phash[g_r + 1:])
49
+ dists = lut[xor].sum(axis=-1)
50
+ for col in np.where(dists <= a.phash_threshold)[0]:
51
+ pairs.add((g_r, g_r + 1 + col))
52
+
53
+ agreed = 0
54
+ disagreed = 0
55
+ by_type = {"concept_concept": [0, 0], "sdxl_sdxl": [0, 0], "cross": [0, 0]}
56
+
57
+ for i, j in pairs:
58
+ s1 = by_id[all_ids[i]]
59
+ s2 = by_id[all_ids[j]]
60
+ match = (s1.harmful == s2.harmful)
61
+
62
+ if match:
63
+ agreed += 1
64
+ else:
65
+ disagreed += 1
66
+
67
+ if s1.source == "concept" and s2.source == "concept":
68
+ key = "concept_concept"
69
+ elif s1.source == "sdxl" and s2.source == "sdxl":
70
+ key = "sdxl_sdxl"
71
+ else:
72
+ key = "cross"
73
+
74
+ by_type[key][0 if match else 1] += 1
75
+
76
+ total = len(pairs)
77
+ print("\n=== LABEL AGREEMENT AMONG NEAR-DUPLICATES ===")
78
+ print(f"Total unique duplicate pairs: {total}")
79
+ print(f"Same label: {agreed:>4} ({agreed / total * 100:.1f}%)")
80
+ print(f"Opposing label: {disagreed:>4} ({disagreed / total * 100:.1f}%)")
81
+ print("\nBreakdown by cohort (Agreed / Disagreed):")
82
+ for k, (agr, dgr) in by_type.items():
83
+ sub_tot = agr + dgr
84
+ pct = (agr / sub_tot * 100) if sub_tot else 0
85
+ print(f" {k:<16}: {agr:>4} agreed, {dgr:>4} disagreed ({pct:.1f}% agreement)")
86
+
87
+
88
+ if __name__ == "__main__":
89
+ main()
@@ -0,0 +1,82 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import json
5
+ from pathlib import Path
6
+ import numpy as np
7
+ from sklearn.metrics import roc_auc_score
8
+
9
+ from .data.vhd11k import load_vhd11k
10
+ from .metrics.scoring import read_predictions
11
+
12
+
13
+ def bootstrap_ci(y_true: np.ndarray, y_score: np.ndarray, n_boot: int = 1000, seed: int = 0) -> tuple[float, float]:
14
+ rng = np.random.default_rng(seed)
15
+ n = len(y_true)
16
+ aucs = []
17
+ for _ in range(n_boot):
18
+ idx = rng.integers(0, n, size=n)
19
+ if len(np.unique(y_true[idx])) < 2:
20
+ continue
21
+ aucs.append(roc_auc_score(y_true[idx], y_score[idx]))
22
+ return float(np.percentile(aucs, 2.5)), float(np.percentile(aucs, 97.5))
23
+
24
+
25
+ def main() -> None:
26
+ ap = argparse.ArgumentParser(prog="python -m disrated.audit_effect")
27
+ ap.add_argument("--dataset", type=Path, required=True)
28
+ ap.add_argument("--predictions", type=Path, default=Path("results/clip-b16-test.jsonl"))
29
+ ap.add_argument("--embeddings", type=Path, default=Path("embeddings/openai_clip-vit-base-patch16.npz"))
30
+ ap.add_argument("--hashes", type=Path, default=Path("embeddings/hashes.npz"))
31
+ ap.add_argument("--splits", type=Path, default=Path("splits.json"))
32
+ ap.add_argument("--cosine-threshold", type=float, required=True)
33
+ a = ap.parse_args()
34
+
35
+ samples, _ = load_vhd11k(a.dataset.expanduser())
36
+ by_id = {s.sample_id: s for s in samples}
37
+ splits = json.loads(a.splits.read_text(encoding="utf-8"))
38
+ preds = read_predictions(a.predictions)
39
+
40
+ z_emb = np.load(a.embeddings, allow_pickle=False)
41
+ id_to_idx = {sid: idx for idx, sid in enumerate(z_emb["ids"].tolist())}
42
+ tr_idx = np.array([id_to_idx[sid] for sid in splits["train"]])
43
+ te_ids = splits["test"]
44
+ te_idx = np.array([id_to_idx[sid] for sid in te_ids])
45
+
46
+ # Cosine check
47
+ X_tr = z_emb["feats"][tr_idx]
48
+ X_te = z_emb["feats"][te_idx]
49
+ max_cos = np.max(np.matmul(X_te, X_tr.T), axis=1)
50
+ cos_leaked = set(np.where(max_cos >= a.cosine_threshold)[0])
51
+
52
+ # pHash check
53
+ z_hash = np.load(a.hashes, allow_pickle=False)
54
+ lut = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
55
+ ph_tr = z_hash["phash"][tr_idx]
56
+ ph_te = z_hash["phash"][te_idx]
57
+
58
+ ph_leaked = set()
59
+ for i in range(len(te_idx)):
60
+ xor = np.bitwise_xor(ph_te[i:i + 1], ph_tr)
61
+ if np.any(lut[xor].sum(axis=-1) <= 8):
62
+ ph_leaked.add(i)
63
+
64
+ leaked_indices = cos_leaked | ph_leaked
65
+ mask_leaked = np.array([i in leaked_indices for i in range(len(te_ids))])
66
+
67
+ y_true = np.array([by_id[sid].harmful for sid in te_ids])
68
+ y_score = np.array([preds[sid].score for sid in te_ids])
69
+
70
+ print("\n=== EFFECT SIZE ON TEST SPLIT ===")
71
+ for label, mask in [("With Train Duplicate", mask_leaked), ("Without Train Duplicate", ~mask_leaked)]:
72
+ n_group = int(np.sum(mask))
73
+ if n_group == 0 or len(np.unique(y_true[mask])) < 2:
74
+ print(f"{label:<28}: n={n_group:>4} | undefined (insufficient class variation)")
75
+ continue
76
+ auc_val = roc_auc_score(y_true[mask], y_score[mask])
77
+ ci_lo, ci_hi = bootstrap_ci(y_true[mask], y_score[mask])
78
+ print(f"{label:<28}: n={n_group:>4} | AUC={auc_val:.4f} [95% CI: {ci_lo:.4f}, {ci_hi:.4f}]")
79
+
80
+
81
+ if __name__ == "__main__":
82
+ main()
@@ -0,0 +1,106 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import json
5
+ from pathlib import Path
6
+ import imagehash
7
+ import numpy as np
8
+
9
+ from .data.vhd11k import load_vhd11k
10
+ from .runner import load_clean
11
+
12
+
13
+ def compute_hashes(samples, cache_path: Path) -> dict[str, np.ndarray]:
14
+ if cache_path.exists():
15
+ z = np.load(cache_path, allow_pickle=False)
16
+ return {"ids": z["ids"], "phash": z["phash"], "dhash": z["dhash"]}
17
+
18
+ ids, ph_bits, dh_bits = [], [], []
19
+ for i, s in enumerate(samples):
20
+ img = load_clean(s.path)
21
+ ids.append(s.sample_id)
22
+ ph_bits.append(imagehash.phash(img).hash.flatten())
23
+ dh_bits.append(imagehash.dhash(img).hash.flatten())
24
+ if (i + 1) % 1000 == 0:
25
+ print(f"hashed {i + 1}/{len(samples)}", flush=True)
26
+
27
+ ids_arr = np.array(ids)
28
+ phash_packed = np.packbits(np.array(ph_bits, dtype=bool), axis=1)
29
+ dhash_packed = np.packbits(np.array(dh_bits, dtype=bool), axis=1)
30
+
31
+ cache_path.parent.mkdir(parents=True, exist_ok=True)
32
+ np.savez_compressed(cache_path, ids=ids_arr, phash=phash_packed, dhash=dhash_packed)
33
+ return {"ids": ids_arr, "phash": phash_packed, "dhash": dhash_packed}
34
+
35
+
36
+ def hamming_chunked(a: np.ndarray, b: np.ndarray, chunk_size: int = 500) -> np.ndarray:
37
+ lut = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
38
+ dists = np.zeros((a.shape[0], b.shape[0]), dtype=np.uint8)
39
+ for start in range(0, a.shape[0], chunk_size):
40
+ end = min(start + chunk_size, a.shape[0])
41
+ xor = np.bitwise_xor(a[start:end, None, :], b[None, :, :])
42
+ dists[start:end] = lut[xor].sum(axis=-1)
43
+ return dists
44
+
45
+
46
+ def audit_hamming(samples, hashes: dict[str, np.ndarray], splits_path: Path) -> dict:
47
+ splits = json.loads(splits_path.read_text(encoding="utf-8"))
48
+ id_to_idx = {sid: idx for idx, sid in enumerate(hashes["ids"].tolist())}
49
+ by_id = {s.sample_id: s for s in samples}
50
+
51
+ split_pairs = [
52
+ ("train", "val", splits["train"], splits["val"]),
53
+ ("train", "test", splits["train"], splits["test"]),
54
+ ("val", "test", splits["val"], splits["test"]),
55
+ ]
56
+
57
+ findings = {}
58
+ for hname in ["phash", "dhash"]:
59
+ print(f"\n=== {hname.upper()} CROSS-SPLIT HAMMING AUDIT ===")
60
+ data = hashes[hname]
61
+ for s1_name, s2_name, ids1, ids2 in split_pairs:
62
+ idx1 = np.array([id_to_idx[sid] for sid in ids1])
63
+ idx2 = np.array([id_to_idx[sid] for sid in ids2])
64
+ dists = hamming_chunked(data[idx1], data[idx2])
65
+
66
+ pair_key = f"{s1_name}_{s2_name}"
67
+ findings.setdefault(hname, {})[pair_key] = {}
68
+
69
+ print(f"\n--- {s1_name} ({len(ids1)}) vs {s2_name} ({len(ids2)}) ---")
70
+ for t in [0, 4, 8]:
71
+ r, c = np.where(dists <= t)
72
+ breakdown = {"concept_concept": 0, "sdxl_sdxl": 0, "cross_source": 0}
73
+ for i_pos, j_pos in zip(r, c):
74
+ s1 = by_id[ids1[i_pos]]
75
+ s2 = by_id[ids2[j_pos]]
76
+ if s1.source == "concept" and s2.source == "concept":
77
+ breakdown["concept_concept"] += 1
78
+ elif s1.source == "sdxl" and s2.source == "sdxl":
79
+ breakdown["sdxl_sdxl"] += 1
80
+ else:
81
+ breakdown["cross_source"] += 1
82
+
83
+ findings[hname][pair_key][t] = {"total": len(r), **breakdown}
84
+ print(
85
+ f"Dist <= {t:>2}: total={len(r):>5} | "
86
+ f"concept-concept={breakdown['concept_concept']:>4} | "
87
+ f"sdxl-sdxl={breakdown['sdxl_sdxl']:>4} | "
88
+ f"cross-source={breakdown['cross_source']:>4}"
89
+ )
90
+ return findings
91
+
92
+
93
+ def main() -> None:
94
+ ap = argparse.ArgumentParser(prog="python -m disrated.audit_hashes")
95
+ ap.add_argument("--dataset", type=Path, required=True)
96
+ ap.add_argument("--splits", type=Path, default=Path("splits.json"))
97
+ ap.add_argument("--cache", type=Path, default=Path("embeddings/hashes.npz"))
98
+ a = ap.parse_args()
99
+
100
+ samples, _ = load_vhd11k(a.dataset.expanduser())
101
+ h = compute_hashes(samples, a.cache)
102
+ audit_hamming(samples, h, a.splits)
103
+
104
+
105
+ if __name__ == "__main__":
106
+ main()
File without changes
@@ -0,0 +1,33 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+
6
+ import numpy as np
7
+ from sklearn.model_selection import StratifiedGroupKFold
8
+
9
+ from .vhd11k import Sample
10
+
11
+
12
+ def make_splits(samples: list[Sample], seed: int = 0, path: Path | None = None) -> dict[str, list[str]]:
13
+ """80/10/10 train/val/test, stratified on source x label, grouped so duplicates never straddle splits.
14
+ If `path` exists it is loaded as-is. Splits are written once and never regenerated."""
15
+ if path is not None and path.exists():
16
+ return json.loads(path.read_text())
17
+
18
+ y = [f"{s.source}|{int(s.harmful)}" for s in samples]
19
+ groups = [s.group for s in samples]
20
+ cv = StratifiedGroupKFold(n_splits=10, shuffle=True, random_state=seed)
21
+ folds = [idx for _, idx in cv.split(np.zeros(len(samples)), y, groups)]
22
+
23
+ def ids(idxs) -> list[str]:
24
+ return sorted(samples[i].sample_id for i in idxs)
25
+
26
+ out = {
27
+ "test": ids(folds[0]),
28
+ "val": ids(folds[1]),
29
+ "train": ids(i for f in folds[2:] for i in f),
30
+ }
31
+ if path is not None:
32
+ path.write_text(json.dumps(out, indent=1))
33
+ return out
@@ -0,0 +1,68 @@
1
+ from __future__ import annotations
2
+
3
+ import csv
4
+ import re
5
+ from pathlib import Path
6
+ from typing import Literal
7
+
8
+ from pydantic import BaseModel, ConfigDict
9
+
10
+ csv.field_size_limit(2**31 - 1) # debate columns are long
11
+
12
+ ANN_FILE = "harmful_image_10000_ann.csv"
13
+ IMG_DIR = "all_10000_evenHarmfulUnharmful"
14
+
15
+ _SDXL = re.compile(r"^sdxl_extend_by_llama3_(harmful|unharmful)_(\d+)_seed(\d+)\.png$")
16
+ _CONCEPT = re.compile(r"^(.+)_(\d+)_(\d+)\.\w+$")
17
+
18
+
19
+ class Sample(BaseModel):
20
+ model_config = ConfigDict(frozen=True)
21
+
22
+ sample_id: str # the CSV imagePath; unique
23
+ path: Path
24
+ harmful: bool # ground truth: the CSV `decision`
25
+ harm_type: str # free-text VLM reason; error analysis only
26
+ source: Literal["sdxl", "concept"]
27
+ concept: str | None = None # one of ~45 tags, concept-named half only
28
+ intent: bool | None = None # what the filename claims (sdxl only). NEVER show to a model.
29
+ group: str # split unit, so near-duplicates stay together
30
+
31
+
32
+ def parse_name(name: str) -> dict:
33
+ if m := _SDXL.match(name):
34
+ intent = m.group(1) == "harmful"
35
+ # same prompt index across seeds -> same group
36
+ return dict(source="sdxl", intent=intent, group=f"sdxl:{m.group(1)}:{m.group(2)}")
37
+ if m := _CONCEPT.match(name):
38
+ # no known duplicate structure here, so each image is its own group
39
+ return dict(source="concept", concept=m.group(1), group=name)
40
+ raise ValueError(f"unrecognised filename: {name}")
41
+
42
+
43
+ def load_vhd11k(root: Path) -> tuple[list[Sample], list[tuple[str, str]]]:
44
+ """Returns (samples, problems). Bad rows are logged and skipped, never fatal."""
45
+ root = Path(root)
46
+ samples: list[Sample] = []
47
+ problems: list[tuple[str, str]] = []
48
+ with (root / ANN_FILE).open(newline="", encoding="utf-8") as f:
49
+ for row in csv.DictReader(f):
50
+ name = row["imagePath"]
51
+ path = root / IMG_DIR / name
52
+ try:
53
+ if row["decision"] not in ("yes", "no"):
54
+ raise ValueError(f"bad decision {row['decision']!r}")
55
+ if not path.is_file() or path.stat().st_size == 0:
56
+ raise ValueError("missing or empty file")
57
+ samples.append(
58
+ Sample(
59
+ sample_id=name,
60
+ path=path,
61
+ harmful=row["decision"] == "yes",
62
+ harm_type=row["harmfulType"],
63
+ **parse_name(name),
64
+ )
65
+ )
66
+ except ValueError as e:
67
+ problems.append((name, str(e)))
68
+ return samples, problems