imgcheck 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
imgcheck/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """imgcheck: perceptual-hash based image deduplication and
2
+ train/test leakage detection for image classification datasets.
3
+ """
4
+
5
+ __version__ = "0.1.0"
6
+ __authors__ = [
7
+ ("Rama Narasimha Kanduri", "kramanarasimha1@gmail.com"),
8
+ ("CS Hari Krishna", "csharikrishna1806@gmail.com"),
9
+ ]
imgcheck/clean.py ADDED
@@ -0,0 +1,262 @@
1
+ """Within-class duplicate cleaning.
2
+
3
+ Copies each unique image (by perceptual hash) into a clean output
4
+ dataset, skipping near-duplicates. Never touches the source dataset —
5
+ always copies, never deletes.
6
+
7
+ Duplicates are found by transitive clustering (see grouping.py) rather
8
+ than a linear "compare against accepted originals" scan, so a chain of
9
+ near-duplicates (A~B, B~C) always collapses into one group with one
10
+ kept image, regardless of file discovery order.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import os
16
+ import shutil
17
+
18
+ import pandas as pd
19
+
20
+ from .grouping import DuplicateCluster, cluster_duplicates
21
+ from .hashing import DEFAULT_EXTENSIONS, HASH_ALGORITHMS, compute_hashes, get_image_paths
22
+
23
+
24
+ def _unique_dst_path(dst_dir: str, filename: str) -> str:
25
+ dst_path = os.path.join(dst_dir, filename)
26
+ if not os.path.exists(dst_path):
27
+ return dst_path
28
+ name, ext = os.path.splitext(filename)
29
+ counter = 1
30
+ while os.path.exists(dst_path):
31
+ dst_path = os.path.join(dst_dir, f"{name}_{counter}{ext}")
32
+ counter += 1
33
+ return dst_path
34
+
35
+
36
+ def clean_duplicates(
37
+ dataset_path: str,
38
+ clean_dataset_path: str,
39
+ report_path: str,
40
+ threshold: int = 3,
41
+ algo: str = "phash",
42
+ extensions=DEFAULT_EXTENSIONS,
43
+ pdf_report_path: str | None = None,
44
+ html_report_path: str | None = None,
45
+ extra_formats: list[str] | None = None,
46
+ ) -> dict:
47
+ """Walk each class folder under dataset_path, cluster perceptually-
48
+ duplicate images together (transitively — see grouping.py), keep
49
+ one representative per cluster, and copy survivors into
50
+ clean_dataset_path (mirroring the class subfolder structure).
51
+
52
+ Writes an Excel report with two sheets:
53
+ - "Duplicate Count": per-class duplicate counts + TOTAL
54
+ - "Matches": every kept/duplicate pair, with a shared Cluster ID
55
+ so transitively-related duplicates can be told
56
+ apart from coincidental pairs
57
+
58
+ Optional outputs:
59
+ - pdf_report_path: visual PDF (requires `fpdf2`, pip install imgcheck[pdf])
60
+ - html_report_path: self-contained HTML gallery grouped by cluster
61
+ - extra_formats: e.g. ["csv", "json"] — also export the two
62
+ report tables in these formats alongside Excel
63
+
64
+ Returns {class_name: duplicate_count}.
65
+ """
66
+ if algo not in HASH_ALGORITHMS:
67
+ raise ValueError(f"Unknown hash algorithm '{algo}'. Choose from {list(HASH_ALGORITHMS)}")
68
+
69
+ os.makedirs(clean_dataset_path, exist_ok=True)
70
+ class_folders = [d for d in os.listdir(dataset_path) if os.path.isdir(os.path.join(dataset_path, d))]
71
+ for cls in class_folders:
72
+ os.makedirs(os.path.join(clean_dataset_path, cls), exist_ok=True)
73
+
74
+ report_rows = []
75
+ duplicate_counts: dict[str, int] = {}
76
+ clusters_by_group: dict[str, list[DuplicateCluster]] = {}
77
+
78
+ for cls in class_folders:
79
+ print("\n" + "=" * 60)
80
+ print("Processing:", cls)
81
+
82
+ cls_path = os.path.join(dataset_path, cls)
83
+ image_paths = get_image_paths(cls_path, extensions)
84
+ print("Total images found:", len(image_paths))
85
+
86
+ hashed_images = compute_hashes(image_paths, algo=algo)
87
+ hashed_by_path = {hi.path: hi for hi in hashed_images}
88
+
89
+ clusters = cluster_duplicates(hashed_images, threshold=threshold)
90
+ clusters_by_group[cls] = clusters
91
+ duplicated_paths = {p for c in clusters for p in c.duplicates}
92
+
93
+ for cluster_id, cluster in enumerate(clusters, start=1):
94
+ kept_hash = hashed_by_path[cluster.kept].hash
95
+ for dup_path in cluster.duplicates:
96
+ dup_hash = hashed_by_path[dup_path].hash
97
+ report_rows.append({
98
+ "Class": cls,
99
+ "Cluster ID": f"{cls}-{cluster_id}",
100
+ "Original Image": os.path.basename(cluster.kept),
101
+ "Duplicate Image": os.path.basename(dup_path),
102
+ "Original Hash": str(kept_hash),
103
+ "Duplicate Hash": str(dup_hash),
104
+ "Hamming Distance": dup_hash - kept_hash,
105
+ # kept out of the Excel export, used only for PDF/HTML thumbnails
106
+ "_Original Path": cluster.kept,
107
+ "_Duplicate Path": dup_path,
108
+ })
109
+
110
+ # Copy every image that is NOT flagged as a duplicate (i.e. every
111
+ # cluster's kept representative, plus every image with no duplicate at all)
112
+ kept_count = 0
113
+ for img_path in image_paths:
114
+ if img_path in duplicated_paths:
115
+ continue
116
+ dst_dir = os.path.join(clean_dataset_path, cls)
117
+ dst_path = _unique_dst_path(dst_dir, os.path.basename(img_path))
118
+ try:
119
+ shutil.copy2(img_path, dst_path)
120
+ kept_count += 1
121
+ except Exception as e:
122
+ print(f"Error copying {img_path}: {e}")
123
+
124
+ duplicate_counts[cls] = len(duplicated_paths)
125
+ print(f"Duplicate groups found: {len(clusters)} | Duplicates removed: {len(duplicated_paths)} | Images kept: {kept_count}")
126
+
127
+ df = pd.DataFrame(report_rows)
128
+ excel_cols = ["Class", "Cluster ID", "Original Image", "Duplicate Image", "Original Hash", "Duplicate Hash", "Hamming Distance"]
129
+ summary_rows = [{"Class": cls, "Duplicate Count": c} for cls, c in duplicate_counts.items()]
130
+ summary_rows.append({"Class": "TOTAL", "Duplicate Count": sum(duplicate_counts.values())})
131
+ summary_df = pd.DataFrame(summary_rows)
132
+ matches_df = df[excel_cols] if not df.empty else pd.DataFrame(columns=excel_cols)
133
+
134
+ os.makedirs(os.path.dirname(report_path) or ".", exist_ok=True)
135
+ with pd.ExcelWriter(report_path, engine="openpyxl") as writer:
136
+ summary_df.to_excel(writer, sheet_name="Duplicate Count", index=False)
137
+ matches_df.to_excel(writer, sheet_name="Matches", index=False)
138
+
139
+ print("\n" + "=" * 60)
140
+ print("DONE")
141
+ print("Clean dataset saved at:", clean_dataset_path)
142
+ print("Duplicate report saved at:", report_path)
143
+
144
+ if extra_formats:
145
+ from .export import export_extra_formats # imported lazily, no hard dependency beyond pandas
146
+ export_extra_formats({"summary": summary_df, "matches": matches_df}, report_path, extra_formats)
147
+
148
+ if pdf_report_path:
149
+ from .report import render_pair_report_pdf # imported lazily: fpdf2 is an optional extra
150
+ render_pair_report_pdf(
151
+ rows=report_rows,
152
+ output_path=pdf_report_path,
153
+ title="Duplicate Removal Report",
154
+ group_key="Class",
155
+ left_label="Kept (original)",
156
+ left_path_key="_Original Path",
157
+ right_label="Removed (duplicate)",
158
+ right_path_key="_Duplicate Path",
159
+ distance_key="Hamming Distance",
160
+ )
161
+ print("Visual PDF report saved at:", pdf_report_path)
162
+
163
+ if html_report_path:
164
+ from .report import render_cluster_report_html
165
+ render_cluster_report_html(
166
+ clusters_by_group=clusters_by_group,
167
+ output_path=html_report_path,
168
+ title="Duplicate Removal Report",
169
+ )
170
+
171
+ return duplicate_counts
172
+
173
+
174
+ def clean_duplicates_inplace(
175
+ dataset_path: str,
176
+ report_path: str,
177
+ threshold: int = 3,
178
+ algo: str = "phash",
179
+ extensions=DEFAULT_EXTENSIONS,
180
+ extra_formats: list[str] | None = None,
181
+ ) -> dict:
182
+ """Remove within-class near-duplicates directly from dataset_path.
183
+
184
+ DESTRUCTIVE: deletes duplicate image files from the source dataset.
185
+ The cluster representative (kept image) is never deleted — only the
186
+ redundant copies are removed.
187
+
188
+ Use clean_duplicates() instead if you want a separate clean copy
189
+ and wish to preserve the original dataset intact.
190
+
191
+ Writes an Excel report with two sheets:
192
+ - 'Duplicate Count': per-class duplicate counts + TOTAL
193
+ - 'Removed Images': every removed duplicate with its kept original
194
+ and Hamming distance
195
+
196
+ Returns {class_name: duplicate_count}.
197
+ """
198
+ if algo not in HASH_ALGORITHMS:
199
+ raise ValueError(f"Unknown hash algorithm '{algo}'. Choose from {list(HASH_ALGORITHMS)}")
200
+
201
+ class_folders = [d for d in os.listdir(dataset_path) if os.path.isdir(os.path.join(dataset_path, d))]
202
+
203
+ report_rows = []
204
+ duplicate_counts: dict[str, int] = {}
205
+
206
+ for cls in class_folders:
207
+ print("\n" + "=" * 60)
208
+ print("Processing:", cls)
209
+
210
+ cls_path = os.path.join(dataset_path, cls)
211
+ image_paths = get_image_paths(cls_path, extensions)
212
+ print("Total images found:", len(image_paths))
213
+
214
+ hashed_images = compute_hashes(image_paths, algo=algo)
215
+ hashed_by_path = {hi.path: hi for hi in hashed_images}
216
+
217
+ clusters = cluster_duplicates(hashed_images, threshold=threshold)
218
+ duplicated_paths = {p for c in clusters for p in c.duplicates}
219
+
220
+ for cluster_id, cluster in enumerate(clusters, start=1):
221
+ kept_hash = hashed_by_path[cluster.kept].hash
222
+ for dup_path in cluster.duplicates:
223
+ dup_hash = hashed_by_path[dup_path].hash
224
+ report_rows.append({
225
+ "Class": cls,
226
+ "Cluster ID": f"{cls}-{cluster_id}",
227
+ "Kept Image": os.path.basename(cluster.kept),
228
+ "Removed Image": os.path.basename(dup_path),
229
+ "Kept Hash": str(kept_hash),
230
+ "Removed Hash": str(dup_hash),
231
+ "Hamming Distance": dup_hash - kept_hash,
232
+ })
233
+ try:
234
+ os.remove(dup_path)
235
+ print(f" Removed: {os.path.basename(dup_path)}")
236
+ except Exception as e:
237
+ print(f" [warn] Could not remove {dup_path}: {e}")
238
+
239
+ duplicate_counts[cls] = len(duplicated_paths)
240
+ print(f"Duplicate groups found: {len(clusters)} | Duplicates removed: {len(duplicated_paths)}")
241
+
242
+ df = pd.DataFrame(report_rows)
243
+ excel_cols = ["Class", "Cluster ID", "Kept Image", "Removed Image", "Kept Hash", "Removed Hash", "Hamming Distance"]
244
+ summary_rows = [{"Class": cls, "Duplicate Count": c} for cls, c in duplicate_counts.items()]
245
+ summary_rows.append({"Class": "TOTAL", "Duplicate Count": sum(duplicate_counts.values())})
246
+ summary_df = pd.DataFrame(summary_rows)
247
+ removed_df = df[excel_cols] if not df.empty else pd.DataFrame(columns=excel_cols)
248
+
249
+ os.makedirs(os.path.dirname(report_path) or ".", exist_ok=True)
250
+ with pd.ExcelWriter(report_path, engine="openpyxl") as writer:
251
+ summary_df.to_excel(writer, sheet_name="Duplicate Count", index=False)
252
+ removed_df.to_excel(writer, sheet_name="Removed Images", index=False)
253
+
254
+ print("\n" + "=" * 60)
255
+ print("DONE — duplicates removed in-place from:", dataset_path)
256
+ print("Report saved at:", report_path)
257
+
258
+ if extra_formats:
259
+ from .export import export_extra_formats
260
+ export_extra_formats({"summary": summary_df, "removed": removed_df}, report_path, extra_formats)
261
+
262
+ return duplicate_counts
imgcheck/cli.py ADDED
@@ -0,0 +1,264 @@
1
+ """Command-line interface for imgcheck.
2
+
3
+ Detection commands (read-only, generate reports):
4
+ imgcheck stats -- dataset health check
5
+ imgcheck clean -- find within-class duplicates, copy survivors to new folder
6
+ imgcheck leakage -- find train/test leaked images
7
+ imgcheck cross-label -- find images mislabelled under two classes
8
+
9
+ Removal commands (modify or copy data, always generate a removal report):
10
+ imgcheck clean-inplace -- delete within-class duplicates in-place
11
+ imgcheck leakage-fix -- remove leaked test images (inplace or copy mode)
12
+ imgcheck cross-label-fix -- remove cross-label duplicates (inplace or copy mode)
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+
19
+ from .clean import clean_duplicates, clean_duplicates_inplace
20
+ from .cross_label import check_cross_label_duplicates
21
+ from .cross_label_fix import fix_cross_label_duplicates
22
+ from .export import SUPPORTED_FORMATS
23
+ from .hashing import HASH_ALGORITHMS
24
+ from .leakage import check_train_test_leakage
25
+ from .leakage_fix import fix_train_test_leakage
26
+ from .stats import compute_dataset_stats
27
+
28
+
29
+ def _add_common_hash_args(parser: argparse.ArgumentParser) -> None:
30
+ parser.add_argument(
31
+ "--threshold", type=int, default=3,
32
+ help="Max Hamming distance to count as a match (0=exact only, 1-3=near-duplicate). Default: 3",
33
+ )
34
+ parser.add_argument(
35
+ "--algo", choices=list(HASH_ALGORITHMS), default="phash",
36
+ help="Perceptual hash algorithm to use. Default: phash",
37
+ )
38
+
39
+
40
+ def _add_report_output_args(parser: argparse.ArgumentParser, pair_or_cluster: str) -> None:
41
+ parser.add_argument(
42
+ "--pdf", default=None,
43
+ help=f"Optional: also write a visual .pdf report of {pair_or_cluster} (needs fpdf2, pip install imgcheck[pdf])",
44
+ )
45
+ parser.add_argument(
46
+ "--html", default=None,
47
+ help=f"Optional: also write a self-contained .html gallery of {pair_or_cluster} (no extra dependency)",
48
+ )
49
+ parser.add_argument(
50
+ "--formats", default=None,
51
+ help=f"Optional: comma-separated extra export formats for the report tables, e.g. csv,json. Choose from {list(SUPPORTED_FORMATS)}",
52
+ )
53
+
54
+
55
+ def _add_fix_mode_args(parser: argparse.ArgumentParser) -> None:
56
+ parser.add_argument(
57
+ "--mode", choices=["inplace", "copy"], default="inplace",
58
+ help=(
59
+ "Removal mode. "
60
+ "'inplace' deletes directly from the source folder (DESTRUCTIVE). "
61
+ "'copy' creates a new clean folder with only the surviving images. "
62
+ "Default: inplace"
63
+ ),
64
+ )
65
+ parser.add_argument(
66
+ "--output", default=None,
67
+ help="Required when --mode copy. Path for the new clean output folder.",
68
+ )
69
+ parser.add_argument(
70
+ "--formats", default=None,
71
+ help=f"Optional: comma-separated extra export formats for the removal report, e.g. csv,json. Choose from {list(SUPPORTED_FORMATS)}",
72
+ )
73
+
74
+
75
+ def _parse_formats(raw: str | None) -> list[str] | None:
76
+ if not raw:
77
+ return None
78
+ return [f.strip() for f in raw.split(",") if f.strip()]
79
+
80
+
81
+ def build_parser() -> argparse.ArgumentParser:
82
+ parser = argparse.ArgumentParser(
83
+ prog="imgcheck",
84
+ description=(
85
+ "Perceptual-hash based image dataset auditing and cleaning. "
86
+ "Detection commands are read-only and generate reports. "
87
+ "Removal commands (*-fix / *-inplace) modify or copy data."
88
+ ),
89
+ )
90
+ subparsers = parser.add_subparsers(dest="command", required=True)
91
+
92
+ # ------------------------------------------------------------------ #
93
+ # DETECTION COMMANDS #
94
+ # ------------------------------------------------------------------ #
95
+
96
+ # --- stats ---
97
+ stats_p = subparsers.add_parser(
98
+ "stats",
99
+ help="Fast dataset health check: counts, resolutions, class balance, corrupt files (no hashing)",
100
+ )
101
+ stats_p.add_argument("--dataset", required=True, help="Path to dataset root (contains class subfolders)")
102
+ stats_p.add_argument("--report", default=None, help="Optional output .xlsx report path")
103
+
104
+ # --- clean ---
105
+ clean_p = subparsers.add_parser(
106
+ "clean",
107
+ help="Find within-class near-duplicates and copy only survivors to a new clean folder (source untouched)",
108
+ )
109
+ clean_p.add_argument("--dataset", required=True, help="Path to source dataset (contains class subfolders)")
110
+ clean_p.add_argument("--output", required=True, help="Path to write the cleaned dataset")
111
+ clean_p.add_argument("--report", required=True, help="Output .xlsx report path")
112
+ _add_report_output_args(clean_p, "kept/duplicate pairs")
113
+ _add_common_hash_args(clean_p)
114
+
115
+ # --- leakage ---
116
+ leak_p = subparsers.add_parser(
117
+ "leakage",
118
+ help="Check for train/test leakage between two dataset splits (read-only, report only)",
119
+ )
120
+ leak_p.add_argument("--train", required=True, help="Path to train root (contains class subfolders)")
121
+ leak_p.add_argument("--test", required=True, help="Path to test root (contains class subfolders)")
122
+ leak_p.add_argument("--report", required=True, help="Output .xlsx report path")
123
+ _add_report_output_args(leak_p, "leaked pairs")
124
+ _add_common_hash_args(leak_p)
125
+
126
+ # --- cross-label ---
127
+ cross_p = subparsers.add_parser(
128
+ "cross-label",
129
+ help="Find images filed under two different class labels (read-only, report only)",
130
+ )
131
+ cross_p.add_argument("--dataset", required=True, help="Path to dataset (contains class subfolders)")
132
+ cross_p.add_argument("--report", required=True, help="Output .xlsx report path")
133
+ _add_report_output_args(cross_p, "cross-label pairs")
134
+ _add_common_hash_args(cross_p)
135
+
136
+ # ------------------------------------------------------------------ #
137
+ # REMOVAL COMMANDS #
138
+ # ------------------------------------------------------------------ #
139
+
140
+ # --- clean-inplace ---
141
+ ci_p = subparsers.add_parser(
142
+ "clean-inplace",
143
+ help="Delete within-class near-duplicates directly from the source dataset (DESTRUCTIVE)",
144
+ )
145
+ ci_p.add_argument("--dataset", required=True, help="Path to dataset root (contains class subfolders)")
146
+ ci_p.add_argument("--report", required=True, help="Output .xlsx removal report path")
147
+ ci_p.add_argument(
148
+ "--formats", default=None,
149
+ help=f"Optional: comma-separated extra export formats, e.g. csv,json. Choose from {list(SUPPORTED_FORMATS)}",
150
+ )
151
+ _add_common_hash_args(ci_p)
152
+
153
+ # --- leakage-fix ---
154
+ lf_p = subparsers.add_parser(
155
+ "leakage-fix",
156
+ help="Detect and remove leaked test images. Use --mode to choose inplace deletion or copy-to-new-folder.",
157
+ )
158
+ lf_p.add_argument("--train", required=True, help="Path to train root (contains class subfolders)")
159
+ lf_p.add_argument("--test", required=True, help="Path to test root (contains class subfolders)")
160
+ lf_p.add_argument("--report", required=True, help="Output .xlsx removal report path")
161
+ _add_fix_mode_args(lf_p)
162
+ _add_common_hash_args(lf_p)
163
+
164
+ # --- cross-label-fix ---
165
+ clf_p = subparsers.add_parser(
166
+ "cross-label-fix",
167
+ help="Detect and remove cross-label duplicate images. Use --mode to choose inplace deletion or copy-to-new-folder.",
168
+ )
169
+ clf_p.add_argument("--dataset", required=True, help="Path to dataset root (contains class subfolders)")
170
+ clf_p.add_argument("--report", required=True, help="Output .xlsx removal report path")
171
+ _add_fix_mode_args(clf_p)
172
+ _add_common_hash_args(clf_p)
173
+
174
+ return parser
175
+
176
+
177
+ def main(argv: list[str] | None = None) -> int:
178
+ parser = build_parser()
179
+ args = parser.parse_args(argv)
180
+
181
+ # --- detection ---
182
+ if args.command == "stats":
183
+ compute_dataset_stats(
184
+ dataset_root=args.dataset,
185
+ report_path=args.report,
186
+ )
187
+
188
+ elif args.command == "clean":
189
+ clean_duplicates(
190
+ dataset_path=args.dataset,
191
+ clean_dataset_path=args.output,
192
+ report_path=args.report,
193
+ threshold=args.threshold,
194
+ algo=args.algo,
195
+ pdf_report_path=args.pdf,
196
+ html_report_path=args.html,
197
+ extra_formats=_parse_formats(args.formats),
198
+ )
199
+
200
+ elif args.command == "leakage":
201
+ check_train_test_leakage(
202
+ train_root=args.train,
203
+ test_root=args.test,
204
+ report_path=args.report,
205
+ threshold=args.threshold,
206
+ algo=args.algo,
207
+ pdf_report_path=args.pdf,
208
+ html_report_path=args.html,
209
+ extra_formats=_parse_formats(args.formats),
210
+ )
211
+
212
+ elif args.command == "cross-label":
213
+ check_cross_label_duplicates(
214
+ dataset_root=args.dataset,
215
+ report_path=args.report,
216
+ threshold=args.threshold,
217
+ algo=args.algo,
218
+ pdf_report_path=args.pdf,
219
+ html_report_path=args.html,
220
+ extra_formats=_parse_formats(args.formats),
221
+ )
222
+
223
+ # --- removal ---
224
+ elif args.command == "clean-inplace":
225
+ clean_duplicates_inplace(
226
+ dataset_path=args.dataset,
227
+ report_path=args.report,
228
+ threshold=args.threshold,
229
+ algo=args.algo,
230
+ extra_formats=_parse_formats(args.formats),
231
+ )
232
+
233
+ elif args.command == "leakage-fix":
234
+ if args.mode == "copy" and not args.output:
235
+ parser.error("--output is required when --mode is 'copy'")
236
+ fix_train_test_leakage(
237
+ train_root=args.train,
238
+ test_root=args.test,
239
+ report_path=args.report,
240
+ mode=args.mode,
241
+ output_root=args.output,
242
+ threshold=args.threshold,
243
+ algo=args.algo,
244
+ extra_formats=_parse_formats(args.formats),
245
+ )
246
+
247
+ elif args.command == "cross-label-fix":
248
+ if args.mode == "copy" and not args.output:
249
+ parser.error("--output is required when --mode is 'copy'")
250
+ fix_cross_label_duplicates(
251
+ dataset_root=args.dataset,
252
+ report_path=args.report,
253
+ mode=args.mode,
254
+ output_root=args.output,
255
+ threshold=args.threshold,
256
+ algo=args.algo,
257
+ extra_formats=_parse_formats(args.formats),
258
+ )
259
+
260
+ return 0
261
+
262
+
263
+ if __name__ == "__main__":
264
+ raise SystemExit(main())
@@ -0,0 +1,134 @@
1
+ """Cross-label duplicate detection.
2
+
3
+ `clean.py` only looks for duplicates *within* a class folder. But a
4
+ more serious bug is the same (or near-identical) image appearing
5
+ under *two different class labels* — e.g. an MRI scan filed under
6
+ both "glioma" and "meningioma". That's not just wasted storage, it's
7
+ a contradictory training signal. This module hashes every image in
8
+ the dataset together (ignoring class) and flags any pair that's
9
+ within threshold but sits in different class folders.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import os
15
+
16
+ import pandas as pd
17
+
18
+ from .hashing import DEFAULT_EXTENSIONS, compute_hashes, get_image_paths, hamming_distance
19
+
20
+
21
+ def check_cross_label_duplicates(
22
+ dataset_root: str,
23
+ report_path: str,
24
+ threshold: int = 3,
25
+ algo: str = "phash",
26
+ extensions=DEFAULT_EXTENSIONS,
27
+ pdf_report_path: str | None = None,
28
+ html_report_path: str | None = None,
29
+ extra_formats: list[str] | None = None,
30
+ ) -> list[dict]:
31
+ """Hash every image under every class subfolder of dataset_root,
32
+ then flag any pair within threshold that belongs to two *different*
33
+ classes (same-class matches are not cross-label issues — see
34
+ clean.py for those).
35
+
36
+ Writes an Excel report with two sheets:
37
+ - "Summary": count of cross-label matches per class pair
38
+ - "Matches": every flagged pair, with both images, both classes,
39
+ and the Hamming distance
40
+
41
+ Optional outputs:
42
+ - pdf_report_path: visual PDF (requires `fpdf2`, pip install imgdedup[pdf])
43
+ - html_report_path: self-contained HTML gallery of flagged pairs
44
+ - extra_formats: e.g. ["csv", "json"] — also export the two
45
+ report tables in these formats alongside Excel
46
+
47
+ Returns the list of matched-pair row dicts.
48
+ """
49
+ class_folders = sorted(d for d in os.listdir(dataset_root) if os.path.isdir(os.path.join(dataset_root, d)))
50
+ if len(class_folders) < 2:
51
+ raise ValueError("Need at least 2 class subfolders to check for cross-label duplicates")
52
+
53
+ all_hashed = [] # list of (HashedImage, class_name)
54
+ for cls in class_folders:
55
+ paths = get_image_paths(os.path.join(dataset_root, cls), extensions)
56
+ for hi in compute_hashes(paths, algo=algo):
57
+ all_hashed.append((hi, cls))
58
+
59
+ print(f"Hashed {len(all_hashed)} images across {len(class_folders)} classes")
60
+
61
+ rows = []
62
+ n = len(all_hashed)
63
+ for i in range(n):
64
+ hi_a, cls_a = all_hashed[i]
65
+ for j in range(i + 1, n):
66
+ hi_b, cls_b = all_hashed[j]
67
+ if cls_a == cls_b:
68
+ continue # same-class matches belong to clean.py, not here
69
+ d = hamming_distance(hi_a.hash, hi_b.hash)
70
+ if d <= threshold:
71
+ rows.append({
72
+ "Class A": cls_a,
73
+ "Image A": os.path.basename(hi_a.path),
74
+ "Class B": cls_b,
75
+ "Image B": os.path.basename(hi_b.path),
76
+ "Hamming Distance": d,
77
+ "_Image A Path": hi_a.path,
78
+ "_Image B Path": hi_b.path,
79
+ })
80
+
81
+ print(f"Cross-label duplicate pairs found: {len(rows)}")
82
+
83
+ df = pd.DataFrame(rows)
84
+ excel_cols = ["Class A", "Image A", "Class B", "Image B", "Hamming Distance"]
85
+
86
+ if not df.empty:
87
+ summary_df = (
88
+ df.groupby(["Class A", "Class B"]).size().reset_index(name="Match Count").sort_values("Match Count", ascending=False)
89
+ )
90
+ else:
91
+ summary_df = pd.DataFrame(columns=["Class A", "Class B", "Match Count"])
92
+
93
+ os.makedirs(os.path.dirname(report_path) or ".", exist_ok=True)
94
+ with pd.ExcelWriter(report_path, engine="openpyxl") as writer:
95
+ summary_df.to_excel(writer, sheet_name="Summary", index=False)
96
+ (df[excel_cols] if not df.empty else df).to_excel(writer, sheet_name="Matches", index=False)
97
+ print("Report saved to:", report_path)
98
+
99
+ matches_export_df = df[excel_cols] if not df.empty else pd.DataFrame(columns=excel_cols)
100
+
101
+ if extra_formats:
102
+ from .export import export_extra_formats
103
+ export_extra_formats({"summary": summary_df, "matches": matches_export_df}, report_path, extra_formats)
104
+
105
+ if pdf_report_path:
106
+ from .report import render_pair_report_pdf
107
+ render_pair_report_pdf(
108
+ rows=rows,
109
+ output_path=pdf_report_path,
110
+ title="Cross-Label Duplicate Report",
111
+ group_key="Class A",
112
+ left_label="Image (Class A)",
113
+ left_path_key="_Image A Path",
114
+ right_label="Image (Class B)",
115
+ right_path_key="_Image B Path",
116
+ distance_key="Hamming Distance",
117
+ )
118
+ print("Visual PDF report saved at:", pdf_report_path)
119
+
120
+ if html_report_path:
121
+ from .report import render_pair_report_html
122
+ render_pair_report_html(
123
+ rows=rows,
124
+ output_path=html_report_path,
125
+ title="Cross-Label Duplicate Report",
126
+ group_key="Class A",
127
+ left_label="Image (Class A)",
128
+ left_path_key="_Image A Path",
129
+ right_label="Image (Class B)",
130
+ right_path_key="_Image B Path",
131
+ distance_key="Hamming Distance",
132
+ )
133
+
134
+ return rows