viseda 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
viseda/image/eda.py ADDED
@@ -0,0 +1,1840 @@
1
+ """
2
+ viseda.image.eda
3
+ ----------------
4
+ Comprehensive EDA for image datasets.
5
+
6
+ Analyses
7
+ ~~~~~~~~
8
+ INVENTORY
9
+ - Total / valid / corrupt count
10
+ - File format & extension distribution
11
+ - File size distribution (KB)
12
+ - Colour-mode distribution (RGB / grayscale / RGBA)
13
+ - Bit-depth distribution
14
+
15
+ SPATIAL
16
+ - Height, width, aspect-ratio distributions
17
+ - Resolution (megapixels) distribution
18
+ - Portrait / landscape / square breakdown
19
+ - Spatial resolution consistency check
20
+
21
+ PIXEL STATISTICS (per image, then aggregated across dataset)
22
+ - Mean, std, min, max per channel
23
+ - Per-channel histograms aggregated across the dataset
24
+ - Global dataset-level pixel mean and std ("dataset statistics" for
25
+ normalisation — the same numbers used in torchvision transforms)
26
+
27
+ COLOUR ANALYSIS
28
+ - RGB, HSV and Lab colour space distributions
29
+ - Dominant colour palette extraction (K-Means across sample)
30
+ - Colour temperature estimate (warm / neutral / cool)
31
+ - Colour cast detection (channel imbalance)
32
+ - Greyscale-like detection (low colour saturation)
33
+
34
+ QUALITY METRICS (per image, then aggregated)
35
+ - Brightness (mean luminance)
36
+ - Contrast (std of luminance)
37
+ - Sharpness (Laplacian variance)
38
+ - Noise estimate (high-freq energy via Laplacian on smooth image)
39
+ - Exposure (over- / under-exposed pixel fraction)
40
+ - Blurriness flag (sharpness below threshold)
41
+ - JPEG compression artefact score (blockiness)
42
+
43
+ TEXTURE & FREQUENCY
44
+ - GLCM texture features: contrast, dissimilarity, homogeneity,
45
+ energy, correlation, ASM (per image → aggregated)
46
+ - FFT frequency energy distribution (low / mid / high freq ratio)
47
+
48
+ DUPLICATE DETECTION
49
+ - Perceptual hash (pHash, 64-bit DCT)
50
+ - Average hash (aHash)
51
+ - Near-duplicate grouping (Hamming distance ≤ threshold)
52
+ - Exact-duplicate detection (MD5)
53
+
54
+ DATASET-LEVEL
55
+ - Class / label distribution (from folder names or dict)
56
+ - Per-class pixel statistics
57
+ - Class imbalance ratio
58
+ - Outlier image detection (images far from dataset mean embedding)
59
+
60
+ VISUALISATIONS
61
+ - Full EDA dashboard (light theme, 6×4 grid)
62
+ - Sample image grid
63
+ - Per-class sample grid
64
+ - Channel correlation matrix
65
+ - Pixel-value heatmap (average image across dataset)
66
+ - UMAP / t-SNE embedding of image features (optional)
67
+ - Duplicate group viewer
68
+ """
69
+
70
+ from __future__ import annotations
71
+
72
+ import hashlib
73
+ import warnings
74
+ from collections import Counter, defaultdict
75
+ from pathlib import Path
76
+ from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
77
+
78
+ import numpy as np
79
+
80
+ # ── lazy heavy imports ────────────────────────────────────────────────────
81
+ def _cv2():
82
+ import cv2; return cv2
83
+
84
+ def _plt():
85
+ import matplotlib.pyplot as plt; return plt
86
+
87
+ def _mpl():
88
+ import matplotlib as mpl; return mpl
89
+
90
+ def _ski_feature():
91
+ from skimage.feature import graycomatrix, graycoprops
92
+ return graycomatrix, graycoprops
93
+
94
+
95
+ # ═════════════════════════════════════════════════════════════════════════════
96
+ # Per-image record
97
+ # ═════════════════════════════════════════════════════════════════════════════
98
+
99
+ class ImageRecord:
100
+ """All per-image statistics stored in one lightweight object."""
101
+
102
+ __slots__ = (
103
+ # identity
104
+ "path", "label", "file_ext", "file_size_kb",
105
+ # spatial
106
+ "height", "width", "channels", "dtype",
107
+ "aspect_ratio", "megapixels",
108
+ # pixel stats
109
+ "mean_rgb", "std_rgb", "min_rgb", "max_rgb",
110
+ # quality
111
+ "brightness", "contrast", "sharpness",
112
+ "noise_estimate",
113
+ "overexposed_frac", "underexposed_frac",
114
+ "is_blurry",
115
+ "compression_score",
116
+ # colour
117
+ "mean_hsv", "mean_lab",
118
+ "saturation_mean",
119
+ "color_temp", # "warm" | "neutral" | "cool"
120
+ "is_grayscale_like",
121
+ # texture
122
+ "glcm_contrast", "glcm_dissimilarity", "glcm_homogeneity",
123
+ "glcm_energy", "glcm_correlation", "glcm_asm",
124
+ # frequency
125
+ "freq_low", "freq_mid", "freq_high",
126
+ # entropy
127
+ "entropy_val",
128
+ # hashes
129
+ "phash", "ahash", "md5",
130
+ # status
131
+ "is_corrupt",
132
+ )
133
+
134
+ def __init__(self):
135
+ for s in self.__slots__:
136
+ setattr(self, s, None)
137
+ self.is_corrupt = False
138
+ self.is_blurry = False
139
+ self.is_grayscale_like = False
140
+
141
+
142
+ # ═════════════════════════════════════════════════════════════════════════════
143
+ # Main class
144
+ # ═════════════════════════════════════════════════════════════════════════════
145
+
146
+ class ImageEDA:
147
+ """
148
+ Comprehensive exploratory data analysis for image datasets.
149
+
150
+ Parameters
151
+ ----------
152
+ verbose : bool
153
+ Print progress messages.
154
+ max_images : int | None
155
+ Analyse at most *max_images* (useful for quick previews).
156
+ n_colors : int
157
+ Dominant-colour clusters to extract (default 8).
158
+ phash_threshold : int
159
+ Hamming distance threshold for near-duplicate detection (default 10).
160
+ blur_threshold : float
161
+ Laplacian variance below this flags an image as blurry (default 50).
162
+ compute_glcm : bool
163
+ Compute GLCM texture features — requires scikit-image (default True).
164
+ compute_freq : bool
165
+ Compute FFT frequency band energies (default True).
166
+
167
+ Examples
168
+ --------
169
+ >>> from viseda import ImageEDA
170
+ >>> eda = ImageEDA()
171
+ >>> eda.load("path/to/dataset/", label_from_parent=True)
172
+ >>> print(eda.summary())
173
+ >>> eda.plot()
174
+ >>> eda.plot_samples()
175
+ >>> eda.plot_average_image()
176
+ >>> eda.report("report.html")
177
+ """
178
+
179
+ SUPPORTED_EXTS = {".jpg", ".jpeg", ".png", ".bmp",
180
+ ".tif", ".tiff", ".webp", ".gif"}
181
+
182
+ def __init__(
183
+ self,
184
+ verbose: bool = True,
185
+ max_images: Optional[int] = None,
186
+ n_colors: int = 8,
187
+ phash_threshold: int = 10,
188
+ blur_threshold: float = 50.0,
189
+ compute_glcm: bool = True,
190
+ compute_freq: bool = True,
191
+ ):
192
+ self.verbose = verbose
193
+ self.max_images = max_images
194
+ self.n_colors = n_colors
195
+ self.phash_threshold = phash_threshold
196
+ self.blur_threshold = blur_threshold
197
+ self.compute_glcm = compute_glcm
198
+ self.compute_freq = compute_freq
199
+
200
+ self._records: List[ImageRecord] = []
201
+ self._labels_map: Dict[str, str] = {}
202
+ self._loaded = False
203
+ self._results: Dict[str, Any] = {}
204
+
205
+ # ─────────────────────────────────────────────────────────────────────
206
+ # Loading
207
+ # ─────────────────────────────────────────────────────────────────────
208
+
209
+ def load(
210
+ self,
211
+ source: Union[str, Path, List],
212
+ labels: Optional[Dict[str, str]] = None,
213
+ label_from_parent: bool = False,
214
+ recursive: bool = True,
215
+ ) -> "ImageEDA":
216
+ """
217
+ Load images from a directory, file, or list of paths.
218
+
219
+ Parameters
220
+ ----------
221
+ source
222
+ Directory path, single image path, or list of paths.
223
+ labels
224
+ ``{path: label}`` mapping.
225
+ label_from_parent
226
+ Infer label from the parent folder name
227
+ (e.g. ``dataset/cats/img.jpg`` → label ``"cats"``).
228
+ recursive
229
+ Recurse into sub-directories (default True).
230
+ """
231
+ paths = self._resolve_paths(source, recursive)
232
+ if self.max_images:
233
+ paths = paths[: self.max_images]
234
+
235
+ if labels:
236
+ self._labels_map = {str(Path(k).resolve()): v
237
+ for k, v in labels.items()}
238
+
239
+ self._log(f"Found {len(paths)} images — computing statistics …")
240
+ self._records = []
241
+
242
+ for i, p in enumerate(paths):
243
+ if self.verbose and i % max(1, len(paths) // 20) == 0:
244
+ self._log(f" [{i:>{len(str(len(paths)))}}/{len(paths)}] {p.name}")
245
+ rec = self._analyse_single(p, label_from_parent)
246
+ self._records.append(rec)
247
+
248
+ self._loaded = True
249
+ n_corrupt = sum(r.is_corrupt for r in self._records)
250
+ self._log(
251
+ f"Done. {len(self._records):,} images loaded "
252
+ f"({n_corrupt} corrupt)."
253
+ )
254
+ return self
255
+
256
+ def load_arrays(
257
+ self,
258
+ arrays: List[np.ndarray],
259
+ labels: Optional[List[str]] = None,
260
+ ) -> "ImageEDA":
261
+ """Load images directly as NumPy arrays (HxWx3 uint8 or float)."""
262
+ self._log(f"Loading {len(arrays)} arrays …")
263
+ self._records = []
264
+ for i, arr in enumerate(arrays):
265
+ rec = ImageRecord()
266
+ rec.path = f"<array_{i}>"
267
+ rec.label = labels[i] if labels and i < len(labels) else None
268
+ rec.file_ext = "array"
269
+ try:
270
+ self._fill_stats(rec, self._to_uint8_rgb(arr))
271
+ except Exception as e:
272
+ rec.is_corrupt = True
273
+ self._log(f" ✗ array_{i}: {e}")
274
+ self._records.append(rec)
275
+ self._loaded = True
276
+ return self
277
+
278
+ # ─────────────────────────────────────────────────────────────────────
279
+ # Summary
280
+ # ─────────────────────────────────────────────────────────────────────
281
+
282
+ def summary(self) -> Dict[str, Any]:
283
+ """
284
+ Return a comprehensive summary dictionary.
285
+
286
+ Sections
287
+ --------
288
+ inventory, spatial, pixel_stats, quality, colour,
289
+ texture, frequency, duplicates, dataset_stats, labels
290
+ """
291
+ self._check_loaded()
292
+ valid = [r for r in self._records if not r.is_corrupt]
293
+ corrupt = [r for r in self._records if r.is_corrupt]
294
+
295
+ if not valid:
296
+ return {"error": "No valid images found."}
297
+
298
+ def arr(attr):
299
+ return np.array([getattr(r, attr) for r in valid
300
+ if getattr(r, attr) is not None])
301
+
302
+ # ── inventory ───────────────────────────────────────────────
303
+ ext_dist = dict(Counter(r.file_ext for r in valid))
304
+ mode_dist = dict(Counter(
305
+ {1: "grayscale", 3: "RGB", 4: "RGBA"}.get(r.channels, str(r.channels))
306
+ for r in valid
307
+ ))
308
+ dtype_dist = dict(Counter(r.dtype for r in valid))
309
+
310
+ # ── spatial ─────────────────────────────────────────────────
311
+ orientations = Counter()
312
+ for r in valid:
313
+ if r.aspect_ratio > 1.05: orientations["landscape"] += 1
314
+ elif r.aspect_ratio < 0.95: orientations["portrait"] += 1
315
+ else: orientations["square"] += 1
316
+
317
+ # ── pixel / colour ───────────────────────────────────────────
318
+ mean_rgb_matrix = np.array([r.mean_rgb for r in valid
319
+ if r.mean_rgb is not None])
320
+ dataset_mean = mean_rgb_matrix.mean(axis=0).tolist() \
321
+ if len(mean_rgb_matrix) else []
322
+ dataset_std_across = mean_rgb_matrix.std(axis=0).tolist() \
323
+ if len(mean_rgb_matrix) else []
324
+
325
+ std_rgb_matrix = np.array([r.std_rgb for r in valid
326
+ if r.std_rgb is not None])
327
+ dataset_pixel_std = std_rgb_matrix.mean(axis=0).tolist() \
328
+ if len(std_rgb_matrix) else []
329
+
330
+ color_temp_dist = dict(Counter(r.color_temp for r in valid
331
+ if r.color_temp))
332
+ grayscale_like_count = sum(r.is_grayscale_like for r in valid)
333
+
334
+ # ── quality ──────────────────────────────────────────────────
335
+ blurry_count = sum(r.is_blurry for r in valid)
336
+ overexp = arr("overexposed_frac")
337
+ underexp = arr("underexposed_frac")
338
+
339
+ # ── texture ──────────────────────────────────────────────────
340
+ texture_summary = {}
341
+ for feat in ("glcm_contrast", "glcm_dissimilarity",
342
+ "glcm_homogeneity", "glcm_energy",
343
+ "glcm_correlation", "glcm_asm"):
344
+ a = arr(feat)
345
+ if len(a):
346
+ texture_summary[feat] = _stat_dict(a)
347
+
348
+ # ── frequency ────────────────────────────────────────────────
349
+ freq_summary = {}
350
+ for band in ("freq_low", "freq_mid", "freq_high"):
351
+ a = arr(band)
352
+ if len(a):
353
+ freq_summary[band] = _stat_dict(a)
354
+
355
+ # ── duplicates ───────────────────────────────────────────────
356
+ exact_dupe_groups = self._find_exact_duplicates(valid)
357
+ near_dupe_groups = self._find_near_duplicates(valid)
358
+
359
+ # ── label info ───────────────────────────────────────────────
360
+ label_dist = None
361
+ class_imbalance_ratio = None
362
+ if any(r.label for r in valid):
363
+ lc = Counter(r.label for r in valid)
364
+ label_dist = dict(lc)
365
+ mx = max(lc.values()); mn = min(lc.values())
366
+ class_imbalance_ratio = round(mx / mn, 3) if mn else None
367
+
368
+ result = {
369
+ # ── inventory
370
+ "inventory": {
371
+ "total": len(self._records),
372
+ "valid": len(valid),
373
+ "corrupt": len(corrupt),
374
+ "corrupt_paths": [r.path for r in corrupt],
375
+ "format_distribution": ext_dist,
376
+ "colour_mode_distribution": mode_dist,
377
+ "dtype_distribution": dtype_dist,
378
+ },
379
+ # ── spatial
380
+ "spatial": {
381
+ "height": _stat_dict(arr("height")),
382
+ "width": _stat_dict(arr("width")),
383
+ "aspect_ratio": _stat_dict(arr("aspect_ratio")),
384
+ "megapixels": _stat_dict(arr("megapixels")),
385
+ "file_size_kb": _stat_dict(arr("file_size_kb")),
386
+ "orientation_distribution": dict(orientations),
387
+ },
388
+ # ── pixel stats
389
+ "pixel_stats": {
390
+ "dataset_mean_rgb": dataset_mean,
391
+ "dataset_std_rgb_across_images": dataset_std_across,
392
+ "dataset_pixel_std_rgb": dataset_pixel_std,
393
+ "per_image_mean_rgb": _stat_dict(mean_rgb_matrix.mean(axis=1))
394
+ if len(mean_rgb_matrix) else {},
395
+ },
396
+ # ── quality
397
+ "quality": {
398
+ "brightness": _stat_dict(arr("brightness")),
399
+ "contrast": _stat_dict(arr("contrast")),
400
+ "sharpness": _stat_dict(arr("sharpness")),
401
+ "noise_estimate": _stat_dict(arr("noise_estimate")),
402
+ "entropy": _stat_dict(arr("entropy_val")),
403
+ "compression_score": _stat_dict(arr("compression_score")),
404
+ "blurry_count": blurry_count,
405
+ "blurry_fraction": round(blurry_count / len(valid), 4),
406
+ "overexposed_frac": _stat_dict(overexp),
407
+ "underexposed_frac": _stat_dict(underexp),
408
+ },
409
+ # ── colour
410
+ "colour": {
411
+ "saturation": _stat_dict(arr("saturation_mean")),
412
+ "colour_temp_distribution": color_temp_dist,
413
+ "grayscale_like_count": grayscale_like_count,
414
+ "grayscale_like_fraction": round(grayscale_like_count / len(valid), 4),
415
+ },
416
+ # ── texture
417
+ "texture": texture_summary,
418
+ # ── frequency
419
+ "frequency": freq_summary,
420
+ # ── duplicates
421
+ "duplicates": {
422
+ "exact_duplicate_groups": exact_dupe_groups,
423
+ "n_exact_duplicate_groups": len(exact_dupe_groups),
424
+ "near_duplicate_groups": near_dupe_groups,
425
+ "n_near_duplicate_groups": len(near_dupe_groups),
426
+ },
427
+ # ── labels
428
+ "labels": {
429
+ "label_distribution": label_dist,
430
+ "class_imbalance_ratio": class_imbalance_ratio,
431
+ },
432
+ }
433
+ self._results["summary"] = result
434
+ return result
435
+
436
+ # ─────────────────────────────────────────────────────────────────────
437
+ # Dataset-level normalisation stats
438
+ # ─────────────────────────────────────────────────────────────────────
439
+
440
+ def normalization_stats(self) -> Dict[str, List[float]]:
441
+ """
442
+ Compute per-channel mean and std for use in dataset normalisation
443
+ (e.g. ``torchvision.transforms.Normalize``).
444
+
445
+ Returns
446
+ -------
447
+ dict with keys ``mean`` and ``std``, each a list of 3 floats
448
+ in [0, 1] (channel order: R, G, B).
449
+
450
+ Example
451
+ -------
452
+ >>> stats = eda.normalization_stats()
453
+ >>> # use in torchvision:
454
+ >>> transforms.Normalize(mean=stats["mean"], std=stats["std"])
455
+ """
456
+ self._check_loaded()
457
+ valid = [r for r in self._records if not r.is_corrupt
458
+ and r.mean_rgb is not None]
459
+ if not valid:
460
+ raise RuntimeError("No valid images to compute statistics from.")
461
+
462
+ means = np.array([r.mean_rgb for r in valid]) / 255.0 # (N, 3)
463
+ stds = np.array([r.std_rgb for r in valid]) / 255.0 # (N, 3)
464
+
465
+ return {
466
+ "mean": means.mean(axis=0).tolist(),
467
+ "std": stds.mean(axis=0).tolist(),
468
+ }
469
+
470
+ # ─────────────────────────────────────────────────────────────────────
471
+ # Plotting — main dashboard
472
+ # ─────────────────────────────────────────────────────────────────────
473
+
474
+ def plot(
475
+ self,
476
+ figsize: Tuple[int, int] = (24, 28),
477
+ save_path: Optional[str] = None,
478
+ dpi: int = 150,
479
+ ) -> None:
480
+ """Render the full EDA dashboard (6-row × 4-col grid)."""
481
+ self._check_loaded()
482
+ plt = _plt(); mpl = _mpl()
483
+ valid = [r for r in self._records if not r.is_corrupt]
484
+
485
+ fig = plt.figure(figsize=figsize, facecolor="white")
486
+ fig.suptitle("VisEDA — Image Dataset Analysis",
487
+ fontsize=24, color="#1f2328", y=0.99, fontweight="bold")
488
+
489
+ gs = mpl.gridspec.GridSpec(
490
+ 6, 4, figure=fig,
491
+ hspace=0.55, wspace=0.35,
492
+ left=0.06, right=0.97, top=0.97, bottom=0.02,
493
+ )
494
+
495
+ def ax(*args, **kw):
496
+ a = fig.add_subplot(*args, **kw)
497
+ a.set_facecolor("#f6f8fa")
498
+ a.tick_params(colors="#57606a", labelsize=8)
499
+ for sp in a.spines.values():
500
+ sp.set_edgecolor("#d0d7de")
501
+ return a
502
+
503
+ # ── Row 0: inventory overview ─────────────────────────────────
504
+ self._plot_info_card(ax(gs[0, :2]), valid)
505
+ self._plot_label_dist(ax(gs[0, 2:]), valid)
506
+
507
+ # ── Row 1: spatial distributions ─────────────────────────────
508
+ self._plot_hist(ax(gs[1, 0]), [r.height for r in valid],
509
+ "Heights (px)", "#58a6ff")
510
+ self._plot_hist(ax(gs[1, 1]), [r.width for r in valid],
511
+ "Widths (px)", "#3fb950")
512
+ self._plot_hist(ax(gs[1, 2]), [r.aspect_ratio for r in valid],
513
+ "Aspect Ratios", "#d2a8ff")
514
+ self._plot_hist(ax(gs[1, 3]), [r.megapixels for r in valid],
515
+ "Megapixels", "#ffa657")
516
+
517
+ # ── Row 2: quality metrics ────────────────────────────────────
518
+ self._plot_hist(ax(gs[2, 0]), [r.brightness for r in valid],
519
+ "Brightness", "#79c0ff")
520
+ self._plot_hist(ax(gs[2, 1]), [r.contrast for r in valid],
521
+ "Contrast (std)", "#56d364")
522
+ self._plot_hist(ax(gs[2, 2]), [r.sharpness for r in valid],
523
+ "Sharpness (Laplacian var)", "#e3b341",
524
+ log_x=True)
525
+ self._plot_hist(ax(gs[2, 3]), [r.noise_estimate for r in valid],
526
+ "Noise Estimate", "#f78166")
527
+
528
+ # ── Row 3: exposure & colour quality ─────────────────────────
529
+ self._plot_hist(ax(gs[3, 0]),
530
+ [r.overexposed_frac * 100 for r in valid],
531
+ "Overexposed Pixels (%)", "#ff6b6b")
532
+ self._plot_hist(ax(gs[3, 1]),
533
+ [r.underexposed_frac * 100 for r in valid],
534
+ "Underexposed Pixels (%)", "#a5d8ff")
535
+ self._plot_hist(ax(gs[3, 2]), [r.saturation_mean for r in valid],
536
+ "Colour Saturation (HSV-S)", "#f9c74f")
537
+ self._plot_hist(ax(gs[3, 3]), [r.entropy_val for r in valid],
538
+ "Pixel Entropy (bits)", "#90be6d")
539
+
540
+ # ── Row 4: channel histograms + texture ──────────────────────
541
+ self._plot_channel_histograms(ax(gs[4, :2]), valid)
542
+ self._plot_texture_radar(ax(gs[4, 2:]), valid)
543
+
544
+ # ── Row 5: frequency + scatter + colour temp ──────────────────
545
+ self._plot_frequency_bands(ax(gs[5, 0]), valid)
546
+ self._plot_brightness_sharpness_scatter(ax(gs[5, 1]), valid)
547
+ self._plot_color_temp_pie(ax(gs[5, 2]), valid)
548
+ self._plot_format_dist(ax(gs[5, 3]), valid)
549
+
550
+ self._finalise(fig, save_path, dpi)
551
+
552
+ # ─────────────────────────────────────────────────────────────────────
553
+ # Plotting — colour dashboard
554
+ # ─────────────────────────────────────────────────────────────────────
555
+
556
+ def plot_colour(
557
+ self,
558
+ figsize: Tuple[int, int] = (22, 14),
559
+ save_path: Optional[str] = None,
560
+ dpi: int = 150,
561
+ ) -> None:
562
+ """Deep-dive colour analysis dashboard."""
563
+ self._check_loaded()
564
+ plt = _plt(); mpl = _mpl()
565
+ valid = [r for r in self._records if not r.is_corrupt]
566
+
567
+ fig = plt.figure(figsize=figsize, facecolor="white")
568
+ fig.suptitle("VisEDA — Colour Analysis",
569
+ fontsize=20, color="#1f2328", y=0.98, fontweight="bold")
570
+
571
+ gs = mpl.gridspec.GridSpec(2, 4, figure=fig,
572
+ hspace=0.45, wspace=0.35,
573
+ left=0.06, right=0.97,
574
+ top=0.93, bottom=0.06)
575
+
576
+ def ax(*args):
577
+ a = fig.add_subplot(*args)
578
+ a.set_facecolor("#f6f8fa")
579
+ a.tick_params(colors="#57606a", labelsize=8)
580
+ for sp in a.spines.values():
581
+ sp.set_edgecolor("#d0d7de")
582
+ return a
583
+
584
+ self._plot_channel_histograms(ax(gs[0, :2]), valid)
585
+ self._plot_dominant_colours(ax(gs[0, 2:]), valid)
586
+ self._plot_hsv_hue_wheel(ax(gs[1, 0]), valid)
587
+ self._plot_saturation_value(ax(gs[1, 1]), valid)
588
+ self._plot_lab_ab_scatter(ax(gs[1, 2]), valid)
589
+ self._plot_color_temp_pie(ax(gs[1, 3]), valid)
590
+
591
+ self._finalise(fig, save_path, dpi)
592
+
593
+ # ─────────────────────────────────────────────────────────────────────
594
+ # Plotting — quality dashboard
595
+ # ─────────────────────────────────────────────────────────────────────
596
+
597
+ def plot_quality(
598
+ self,
599
+ figsize: Tuple[int, int] = (22, 14),
600
+ save_path: Optional[str] = None,
601
+ dpi: int = 150,
602
+ ) -> None:
603
+ """Deep-dive quality metrics dashboard."""
604
+ self._check_loaded()
605
+ plt = _plt(); mpl = _mpl()
606
+ valid = [r for r in self._records if not r.is_corrupt]
607
+
608
+ fig = plt.figure(figsize=figsize, facecolor="white")
609
+ fig.suptitle("VisEDA — Quality Analysis",
610
+ fontsize=20, color="#1f2328", y=0.98, fontweight="bold")
611
+
612
+ gs = mpl.gridspec.GridSpec(2, 4, figure=fig,
613
+ hspace=0.45, wspace=0.35,
614
+ left=0.06, right=0.97,
615
+ top=0.93, bottom=0.06)
616
+
617
+ def ax(*args):
618
+ a = fig.add_subplot(*args)
619
+ a.set_facecolor("#f6f8fa")
620
+ a.tick_params(colors="#57606a", labelsize=8)
621
+ for sp in a.spines.values():
622
+ sp.set_edgecolor("#d0d7de")
623
+ return a
624
+
625
+ self._plot_hist(ax(gs[0, 0]), [r.sharpness for r in valid],
626
+ "Sharpness", "#e3b341", log_x=True)
627
+ self._plot_hist(ax(gs[0, 1]), [r.noise_estimate for r in valid],
628
+ "Noise Estimate", "#f78166")
629
+ self._plot_hist(ax(gs[0, 2]),
630
+ [r.compression_score for r in valid
631
+ if r.compression_score is not None],
632
+ "JPEG Blockiness Score", "#79c0ff")
633
+ self._plot_blurry_breakdown(ax(gs[0, 3]), valid)
634
+ self._plot_exposure_scatter(ax(gs[1, :2]), valid)
635
+ self._plot_texture_radar(ax(gs[1, 2:]), valid)
636
+
637
+ self._finalise(fig, save_path, dpi)
638
+
639
+ # ─────────────────────────────────────────────────────────────────────
640
+ # Plotting — texture dashboard
641
+ # ─────────────────────────────────────────────────────────────────────
642
+
643
+ def plot_texture(
644
+ self,
645
+ figsize: Tuple[int, int] = (22, 10),
646
+ save_path: Optional[str] = None,
647
+ dpi: int = 150,
648
+ ) -> None:
649
+ """GLCM texture features and frequency analysis dashboard."""
650
+ self._check_loaded()
651
+ plt = _plt(); mpl = _mpl()
652
+ valid = [r for r in self._records if not r.is_corrupt]
653
+
654
+ fig = plt.figure(figsize=figsize, facecolor="white")
655
+ fig.suptitle("VisEDA — Texture & Frequency Analysis",
656
+ fontsize=20, color="#1f2328", y=0.98, fontweight="bold")
657
+
658
+ gs = mpl.gridspec.GridSpec(1, 4, figure=fig,
659
+ hspace=0.4, wspace=0.35,
660
+ left=0.06, right=0.97,
661
+ top=0.88, bottom=0.1)
662
+
663
+ def ax(*args):
664
+ a = fig.add_subplot(*args)
665
+ a.set_facecolor("#f6f8fa")
666
+ a.tick_params(colors="#57606a", labelsize=8)
667
+ for sp in a.spines.values():
668
+ sp.set_edgecolor("#d0d7de")
669
+ return a
670
+
671
+ for i, (feat, color) in enumerate([
672
+ ("glcm_contrast", "#58a6ff"),
673
+ ("glcm_homogeneity", "#3fb950"),
674
+ ("glcm_energy", "#e3b341"),
675
+ ("glcm_correlation", "#d2a8ff"),
676
+ ]):
677
+ self._plot_hist(ax(gs[0, i]),
678
+ [getattr(r, feat) for r in valid
679
+ if getattr(r, feat) is not None],
680
+ feat.replace("_", " ").title(), color)
681
+
682
+ self._finalise(fig, save_path, dpi)
683
+
684
+ # ─────────────────────────────────────────────────────────────────────
685
+ # Plotting — sample grid
686
+ # ─────────────────────────────────────────────────────────────────────
687
+
688
+ def plot_samples(
689
+ self,
690
+ n: int = 25,
691
+ cols: int = 5,
692
+ label: Optional[str] = None,
693
+ random_seed: int = 42,
694
+ figsize: Optional[Tuple] = None,
695
+ save_path: Optional[str] = None,
696
+ ) -> None:
697
+ """
698
+ Display a grid of sample images.
699
+
700
+ Parameters
701
+ ----------
702
+ label
703
+ If supplied, only show images with this label.
704
+ """
705
+ self._check_loaded()
706
+ plt = _plt(); cv2 = _cv2()
707
+
708
+ pool = [r for r in self._records
709
+ if not r.is_corrupt and not r.path.startswith("<array")]
710
+ if label:
711
+ pool = [r for r in pool if r.label == label]
712
+
713
+ rng = np.random.default_rng(random_seed)
714
+ sample = rng.choice(pool, size=min(n, len(pool)), replace=False)
715
+ rows = int(np.ceil(len(sample) / cols))
716
+ figsize = figsize or (cols * 3, rows * 3 + 0.5)
717
+
718
+ fig, axes = plt.subplots(rows, cols, figsize=figsize,
719
+ facecolor="white")
720
+ axes = np.array(axes).flatten()
721
+
722
+ for i, rec in enumerate(sample):
723
+ img = cv2.imread(str(rec.path))
724
+ if img is None:
725
+ axes[i].axis("off"); continue
726
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
727
+ img = _resize_display(img, 224)
728
+ axes[i].imshow(img)
729
+ title = (rec.label or Path(rec.path).stem)[:20]
730
+ axes[i].set_title(title, fontsize=7, color="#1f2328")
731
+ axes[i].axis("off")
732
+
733
+ for j in range(len(sample), len(axes)):
734
+ axes[j].axis("off")
735
+
736
+ title_str = f"Sample Images" + (f" — {label}" if label else "")
737
+ fig.suptitle(title_str, color="#1f2328", fontsize=13)
738
+ plt.tight_layout()
739
+ self._finalise(fig, save_path, dpi=120)
740
+
741
+ # ─────────────────────────────────────────────────────────────────────
742
+ # Plotting — per-class sample grids
743
+ # ─────────────────────────────────────────────────────────────────────
744
+
745
+ def plot_class_samples(
746
+ self,
747
+ n_per_class: int = 5,
748
+ save_path: Optional[str] = None,
749
+ ) -> None:
750
+ """One row of sample images per class label."""
751
+ self._check_loaded()
752
+ plt = _plt(); cv2 = _cv2()
753
+
754
+ classes = sorted(set(r.label for r in self._records
755
+ if r.label and not r.is_corrupt))
756
+ if not classes:
757
+ self._log("No labels found — use label_from_parent=True when loading.")
758
+ return
759
+
760
+ n_classes = len(classes)
761
+ fig, axes = plt.subplots(
762
+ n_classes, n_per_class,
763
+ figsize=(n_per_class * 2.5, n_classes * 2.5),
764
+ facecolor="white",
765
+ )
766
+ if n_classes == 1:
767
+ axes = axes[np.newaxis, :]
768
+
769
+ rng = np.random.default_rng(0)
770
+ for row, cls in enumerate(classes):
771
+ pool = [r for r in self._records
772
+ if r.label == cls and not r.is_corrupt
773
+ and not r.path.startswith("<array")]
774
+ sample = rng.choice(pool, size=min(n_per_class, len(pool)),
775
+ replace=False)
776
+ for col in range(n_per_class):
777
+ ax = axes[row, col]
778
+ ax.axis("off")
779
+ ax.set_facecolor("white")
780
+ if col >= len(sample):
781
+ continue
782
+ img = cv2.imread(str(sample[col].path))
783
+ if img is None:
784
+ continue
785
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
786
+ img = _resize_display(img, 128)
787
+ ax.imshow(img)
788
+ if col == 0:
789
+ ax.set_ylabel(cls, color="#1f2328", fontsize=8,
790
+ rotation=0, labelpad=50, va="center")
791
+
792
+ fig.suptitle("Per-class Sample Images", color="#1f2328", fontsize=14)
793
+ plt.tight_layout()
794
+ self._finalise(fig, save_path, dpi=120)
795
+
796
+ # ─────────────────────────────────────────────────────────────────────
797
+ # Plotting — average image
798
+ # ─────────────────────────────────────────────────────────────────────
799
+
800
+ def plot_average_image(
801
+ self,
802
+ target_size: Tuple[int, int] = (224, 224),
803
+ label: Optional[str] = None,
804
+ save_path: Optional[str] = None,
805
+ ) -> None:
806
+ """
807
+ Compute and display the pixel-wise average image across the dataset.
808
+
809
+ This reveals systematic dataset biases (e.g. sky always at top,
810
+ ground at bottom, objects centred).
811
+ """
812
+ self._check_loaded()
813
+ plt = _plt(); cv2 = _cv2()
814
+
815
+ pool = [r for r in self._records
816
+ if not r.is_corrupt and not r.path.startswith("<array")]
817
+ if label:
818
+ pool = [r for r in pool if r.label == label]
819
+
820
+ if not pool:
821
+ self._log("No images available for average-image computation.")
822
+ return
823
+
824
+ self._log(f"Computing average image over {len(pool)} images …")
825
+ acc = np.zeros((*target_size, 3), dtype=np.float64)
826
+ count = 0
827
+ for rec in pool:
828
+ img = cv2.imread(str(rec.path))
829
+ if img is None:
830
+ continue
831
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
832
+ img = cv2.resize(img, (target_size[1], target_size[0]))
833
+ acc += img.astype(np.float64)
834
+ count += 1
835
+
836
+ if count == 0:
837
+ self._log("No images could be read.")
838
+ return
839
+
840
+ avg = (acc / count).astype(np.uint8)
841
+
842
+ fig, axes = plt.subplots(1, 2, figsize=(10, 5), facecolor="white")
843
+ axes[0].imshow(avg)
844
+ axes[0].set_title("Average Image", color="#1f2328", fontsize=12)
845
+ axes[0].axis("off")
846
+
847
+ # standard deviation image
848
+ self._log("Computing std-dev image …")
849
+ sq_acc = np.zeros_like(acc)
850
+ for rec in pool:
851
+ img = cv2.imread(str(rec.path))
852
+ if img is None:
853
+ continue
854
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
855
+ img = cv2.resize(img, (target_size[1], target_size[0]))
856
+ sq_acc += (img.astype(np.float64) - acc / count) ** 2
857
+
858
+ std_img = np.sqrt(sq_acc / count).astype(np.uint8)
859
+ axes[1].imshow(std_img)
860
+ axes[1].set_title("Std-Dev Image\n(high = high variance across dataset)",
861
+ color="#1f2328", fontsize=12)
862
+ axes[1].axis("off")
863
+
864
+ title = f"Dataset Average & Variance" + (f" — {label}" if label else "")
865
+ fig.suptitle(title, color="#1f2328", fontsize=14)
866
+ plt.tight_layout()
867
+ self._finalise(fig, save_path, dpi=150)
868
+
869
+ # ─────────────────────────────────────────────────────────────────────
870
+ # Plotting — channel correlation
871
+ # ─────────────────────────────────────────────────────────────────────
872
+
873
+ def plot_channel_correlation(
874
+ self,
875
+ max_pixels: int = 50_000,
876
+ save_path: Optional[str] = None,
877
+ ) -> None:
878
+ """R/G/B pairwise scatter matrix across a random pixel sample."""
879
+ self._check_loaded()
880
+ plt = _plt(); cv2 = _cv2()
881
+
882
+ pool = [r for r in self._records if not r.is_corrupt
883
+ and not r.path.startswith("<array")][:80]
884
+
885
+ rng = np.random.default_rng(0)
886
+ pixels = []
887
+ for rec in pool:
888
+ img = cv2.imread(str(rec.path))
889
+ if img is None:
890
+ continue
891
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
892
+ flat = img.reshape(-1, 3)
893
+ n = max_pixels // len(pool)
894
+ if len(flat) > n:
895
+ flat = flat[rng.choice(len(flat), n, replace=False)]
896
+ pixels.append(flat)
897
+
898
+ if not pixels:
899
+ return
900
+ pixels = np.vstack(pixels).astype(float)
901
+ ch_names = ["Red", "Green", "Blue"]
902
+ ch_colors = ["#ff6b6b", "#51cf66", "#339af0"]
903
+
904
+ fig, axes = plt.subplots(3, 3, figsize=(11, 11), facecolor="white")
905
+ for i in range(3):
906
+ for j in range(3):
907
+ a = axes[i, j]
908
+ a.set_facecolor("#f6f8fa")
909
+ for sp in a.spines.values():
910
+ sp.set_edgecolor("#d0d7de")
911
+ a.tick_params(colors="#57606a", labelsize=7)
912
+ if i == j:
913
+ a.hist(pixels[:, i], bins=60, color=ch_colors[i], alpha=0.85)
914
+ a.set_title(ch_names[i], color="#1f2328", fontsize=9)
915
+ else:
916
+ a.scatter(pixels[:, j], pixels[:, i],
917
+ alpha=0.04, s=1, color=ch_colors[i])
918
+ corr = float(np.corrcoef(pixels[:, i], pixels[:, j])[0, 1])
919
+ a.set_title(f"r = {corr:.3f}", color="#57606a", fontsize=8)
920
+ if i == 2:
921
+ a.set_xlabel(ch_names[j], color="#57606a", fontsize=8)
922
+ if j == 0:
923
+ a.set_ylabel(ch_names[i], color="#57606a", fontsize=8)
924
+
925
+ fig.suptitle("Channel Correlation Matrix", color="#1f2328", fontsize=14)
926
+ plt.tight_layout()
927
+ self._finalise(fig, save_path, dpi=130)
928
+
929
+ # ─────────────────────────────────────────────────────────────────────
930
+ # Plotting — duplicate viewer
931
+ # ─────────────────────────────────────────────────────────────────────
932
+
933
+ def plot_duplicates(
934
+ self,
935
+ mode: str = "near",
936
+ max_groups: int = 5,
937
+ save_path: Optional[str] = None,
938
+ ) -> None:
939
+ """
940
+ Visualise duplicate / near-duplicate groups.
941
+
942
+ Parameters
943
+ ----------
944
+ mode : ``"exact"`` | ``"near"``
945
+ """
946
+ self._check_loaded()
947
+ plt = _plt(); cv2 = _cv2()
948
+
949
+ valid = [r for r in self._records if not r.is_corrupt]
950
+ groups = (self._find_exact_duplicates(valid) if mode == "exact"
951
+ else self._find_near_duplicates(valid))
952
+ groups = groups[:max_groups]
953
+
954
+ if not groups:
955
+ self._log(f"No {mode} duplicates found.")
956
+ return
957
+
958
+ max_cols = max(len(g) for g in groups)
959
+ fig, axes = plt.subplots(
960
+ len(groups), max_cols,
961
+ figsize=(max_cols * 2.5, len(groups) * 2.5),
962
+ facecolor="white",
963
+ squeeze=False,
964
+ )
965
+
966
+ for row, group in enumerate(groups):
967
+ for col in range(max_cols):
968
+ ax = axes[row, col]
969
+ ax.axis("off")
970
+ ax.set_facecolor("white")
971
+ if col >= len(group):
972
+ continue
973
+ img = cv2.imread(str(group[col]))
974
+ if img is None:
975
+ continue
976
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
977
+ img = _resize_display(img, 128)
978
+ ax.imshow(img)
979
+ ax.set_title(Path(group[col]).name[:18],
980
+ fontsize=6, color="#57606a")
981
+
982
+ fig.suptitle(f"{mode.capitalize()} Duplicate Groups",
983
+ color="#1f2328", fontsize=13)
984
+ plt.tight_layout()
985
+ self._finalise(fig, save_path, dpi=120)
986
+
987
+ # ─────────────────────────────────────────────────────────────────────
988
+ # Report
989
+ # ─────────────────────────────────────────────────────────────────────
990
+
991
+ def report(self, output_path: str = "viseda_report.html") -> str:
992
+ """Generate a self-contained HTML report."""
993
+ self._check_loaded()
994
+ s = self.summary()
995
+ norm = self.normalization_stats()
996
+ s["normalization_stats"] = norm
997
+ _generate_html_report(s, output_path)
998
+ self._log(f"Report saved → {output_path}")
999
+ return output_path
1000
+
1001
+ # ─────────────────────────────────────────────────────────────────────
1002
+ # Per-image analysis
1003
+ # ─────────────────────────────────────────────────────────────────────
1004
+
1005
+ def _analyse_single(self, path: Path, label_from_parent: bool) -> ImageRecord:
1006
+ cv2 = _cv2()
1007
+ rec = ImageRecord()
1008
+ rec.path = str(path)
1009
+ rec.file_ext = path.suffix.lower()
1010
+ rec.file_size_kb = path.stat().st_size / 1024
1011
+
1012
+ if label_from_parent:
1013
+ rec.label = path.parent.name
1014
+ else:
1015
+ rec.label = self._labels_map.get(str(path.resolve()))
1016
+
1017
+ img_bgr = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
1018
+ if img_bgr is None:
1019
+ rec.is_corrupt = True
1020
+ return rec
1021
+
1022
+ # Normalise to HxWx3 uint8
1023
+ if img_bgr.ndim == 2:
1024
+ img_bgr = cv2.cvtColor(img_bgr, cv2.COLOR_GRAY2BGR)
1025
+ elif img_bgr.shape[2] == 4:
1026
+ img_bgr = cv2.cvtColor(img_bgr, cv2.COLOR_BGRA2BGR)
1027
+
1028
+ img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
1029
+ self._fill_stats(rec, img_rgb)
1030
+ return rec
1031
+
1032
+ def _fill_stats(self, rec: ImageRecord, img: np.ndarray) -> None:
1033
+ cv2 = _cv2()
1034
+
1035
+ if img.ndim == 2:
1036
+ img = np.stack([img, img, img], axis=-1)
1037
+ img = img.astype(np.uint8)
1038
+
1039
+ h, w = img.shape[:2]
1040
+ rec.height = h
1041
+ rec.width = w
1042
+ rec.channels = img.shape[2] if img.ndim == 3 else 1
1043
+ rec.dtype = str(img.dtype)
1044
+ rec.aspect_ratio = round(w / h, 4)
1045
+ rec.megapixels = round(h * w / 1_000_000, 4)
1046
+
1047
+ f = img.astype(np.float32)
1048
+
1049
+ # ── pixel stats ─────────────────────────────────────────────
1050
+ rec.mean_rgb = f.mean(axis=(0, 1)).tolist()
1051
+ rec.std_rgb = f.std(axis=(0, 1)).tolist()
1052
+ rec.min_rgb = f.min(axis=(0, 1)).tolist()
1053
+ rec.max_rgb = f.max(axis=(0, 1)).tolist()
1054
+
1055
+ # ── brightness & contrast (luminance) ────────────────────────
1056
+ gray = (0.299 * f[:, :, 0] + 0.587 * f[:, :, 1]
1057
+ + 0.114 * f[:, :, 2])
1058
+ rec.brightness = float(gray.mean())
1059
+ rec.contrast = float(gray.std())
1060
+
1061
+ # ── sharpness: Laplacian variance ────────────────────────────
1062
+ gray_u8 = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
1063
+ lap = cv2.Laplacian(gray_u8, cv2.CV_64F)
1064
+ rec.sharpness = float(lap.var())
1065
+ rec.is_blurry = rec.sharpness < self.blur_threshold
1066
+
1067
+ # ── noise estimate: high-freq residual ───────────────────────
1068
+ blurred = cv2.GaussianBlur(gray_u8, (5, 5), 0).astype(np.float32)
1069
+ residual = gray_u8.astype(np.float32) - blurred
1070
+ rec.noise_estimate = float(residual.std())
1071
+
1072
+ # ── exposure ─────────────────────────────────────────────────
1073
+ rec.overexposed_frac = float((gray > 245).mean())
1074
+ rec.underexposed_frac = float((gray < 10).mean())
1075
+
1076
+ # ── entropy ──────────────────────────────────────────────────
1077
+ hist, _ = np.histogram(gray_u8.ravel(), bins=256, range=(0, 256))
1078
+ p = hist / (hist.sum() + 1e-12)
1079
+ p = p[p > 0]
1080
+ rec.entropy_val = float(-np.sum(p * np.log2(p)))
1081
+
1082
+ # ── JPEG blockiness score ────────────────────────────────────
1083
+ rec.compression_score = self._blockiness(gray_u8)
1084
+
1085
+ # ── colour spaces ────────────────────────────────────────────
1086
+ hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV).astype(np.float32)
1087
+ rec.mean_hsv = hsv.mean(axis=(0, 1)).tolist()
1088
+ rec.saturation_mean = float(hsv[:, :, 1].mean())
1089
+ rec.is_grayscale_like = rec.saturation_mean < 15.0
1090
+
1091
+ lab = cv2.cvtColor(img, cv2.COLOR_RGB2Lab).astype(np.float32)
1092
+ rec.mean_lab = lab.mean(axis=(0, 1)).tolist()
1093
+
1094
+ # colour temperature from R/B ratio
1095
+ r_mean = rec.mean_rgb[0]; b_mean = rec.mean_rgb[2]
1096
+ ratio = r_mean / (b_mean + 1e-6)
1097
+ if ratio > 1.1: rec.color_temp = "warm"
1098
+ elif ratio < 0.9: rec.color_temp = "cool"
1099
+ else: rec.color_temp = "neutral"
1100
+
1101
+ # ── perceptual hash (pHash) ──────────────────────────────────
1102
+ small = cv2.resize(gray_u8, (32, 32),
1103
+ interpolation=cv2.INTER_AREA).astype(np.float32)
1104
+ dct = cv2.dct(small)
1105
+ dct_low = dct[:8, :8].flatten()
1106
+ med = np.median(dct_low)
1107
+ rec.phash = int("".join("1" if v > med else "0"
1108
+ for v in dct_low), 2)
1109
+
1110
+ # average hash (aHash)
1111
+ tiny = cv2.resize(gray_u8, (8, 8),
1112
+ interpolation=cv2.INTER_AREA).astype(np.float32)
1113
+ mean_tiny = tiny.mean()
1114
+ rec.ahash = int("".join("1" if v > mean_tiny else "0"
1115
+ for v in tiny.flatten()), 2)
1116
+
1117
+ # MD5 (for exact duplicates)
1118
+ raw = img.tobytes()
1119
+ rec.md5 = hashlib.md5(raw).hexdigest()
1120
+
1121
+ # ── GLCM texture ─────────────────────────────────────────────
1122
+ if self.compute_glcm:
1123
+ self._fill_glcm(rec, gray_u8)
1124
+
1125
+ # ── FFT frequency bands ───────────────────────────────────────
1126
+ if self.compute_freq:
1127
+ self._fill_frequency(rec, gray_u8)
1128
+
1129
+ def _fill_glcm(self, rec: ImageRecord, gray_u8: np.ndarray) -> None:
1130
+ try:
1131
+ graycomatrix, graycoprops = _ski_feature()
1132
+ # Downsample for speed
1133
+ small = _cv2().resize(gray_u8, (128, 128))
1134
+ # Quantise to 64 levels
1135
+ quant = (small // 4).astype(np.uint8)
1136
+ glcm = graycomatrix(quant, distances=[1],
1137
+ angles=[0, np.pi / 4, np.pi / 2, 3 * np.pi / 4],
1138
+ levels=64, symmetric=True, normed=True)
1139
+ rec.glcm_contrast = float(graycoprops(glcm, "contrast").mean())
1140
+ rec.glcm_dissimilarity = float(graycoprops(glcm, "dissimilarity").mean())
1141
+ rec.glcm_homogeneity = float(graycoprops(glcm, "homogeneity").mean())
1142
+ rec.glcm_energy = float(graycoprops(glcm, "energy").mean())
1143
+ rec.glcm_correlation = float(graycoprops(glcm, "correlation").mean())
1144
+ rec.glcm_asm = float(graycoprops(glcm, "ASM").mean())
1145
+ except Exception:
1146
+ pass
1147
+
1148
+ def _fill_frequency(self, rec: ImageRecord, gray_u8: np.ndarray) -> None:
1149
+ try:
1150
+ small = _cv2().resize(gray_u8, (128, 128)).astype(np.float32)
1151
+ fft = np.fft.fft2(small)
1152
+ fft_s = np.fft.fftshift(np.abs(fft))
1153
+ h, w = fft_s.shape
1154
+ cy, cx = h // 2, w // 2
1155
+ total = fft_s.sum() + 1e-9
1156
+
1157
+ # Low = inner 10 %, Mid = 10–40 %, High = outer 40–50 %
1158
+ Y, X = np.ogrid[:h, :w]
1159
+ dist = np.sqrt((Y - cy) ** 2 + (X - cx) ** 2)
1160
+ max_d = min(cy, cx)
1161
+
1162
+ rec.freq_low = float(fft_s[dist < max_d * 0.10].sum() / total)
1163
+ rec.freq_mid = float(fft_s[(dist >= max_d * 0.10)
1164
+ & (dist < max_d * 0.40)].sum() / total)
1165
+ rec.freq_high = float(fft_s[dist >= max_d * 0.40].sum() / total)
1166
+ except Exception:
1167
+ pass
1168
+
1169
+ @staticmethod
1170
+ def _blockiness(gray_u8: np.ndarray) -> float:
1171
+ """Estimate JPEG blockiness as mean absolute difference across 8-px boundaries."""
1172
+ h, w = gray_u8.shape
1173
+ g = gray_u8.astype(np.float32)
1174
+ scores = []
1175
+ for y in range(8, h, 8):
1176
+ scores.append(np.abs(g[y, :] - g[y - 1, :]).mean())
1177
+ for x in range(8, w, 8):
1178
+ scores.append(np.abs(g[:, x] - g[:, x - 1]).mean())
1179
+ return float(np.mean(scores)) if scores else 0.0
1180
+
1181
+ # ─────────────────────────────────────────────────────────────────────
1182
+ # Duplicate detection
1183
+ # ─────────────────────────────────────────────────────────────────────
1184
+
1185
+ def _find_exact_duplicates(
1186
+ self, records: List[ImageRecord]
1187
+ ) -> List[List[str]]:
1188
+ groups: Dict[str, List[str]] = defaultdict(list)
1189
+ for r in records:
1190
+ if r.md5:
1191
+ groups[r.md5].append(r.path)
1192
+ return [g for g in groups.values() if len(g) > 1]
1193
+
1194
+ def _find_near_duplicates(
1195
+ self, records: List[ImageRecord]
1196
+ ) -> List[List[str]]:
1197
+ groups: List[List[str]] = []
1198
+ visited: set = set()
1199
+ hashes = [(r.phash, r.path) for r in records if r.phash is not None]
1200
+ for i, (h1, p1) in enumerate(hashes):
1201
+ if p1 in visited:
1202
+ continue
1203
+ grp = [p1]
1204
+ for h2, p2 in hashes[i + 1:]:
1205
+ if p2 in visited:
1206
+ continue
1207
+ if bin(h1 ^ h2).count("1") <= self.phash_threshold:
1208
+ grp.append(p2)
1209
+ visited.add(p2)
1210
+ if len(grp) > 1:
1211
+ groups.append(grp)
1212
+ visited.add(p1)
1213
+ return groups
1214
+
1215
+ # ─────────────────────────────────────────────────────────────────────
1216
+ # Plot helpers
1217
+ # ─────────────────────────────────────────────────────────────────────
1218
+
1219
+ def _plot_info_card(self, ax, valid):
1220
+ ax.axis("off")
1221
+ corrupt = len(self._records) - len(valid)
1222
+ blurry = sum(r.is_blurry for r in valid)
1223
+ grey_like = sum(r.is_grayscale_like for r in valid)
1224
+ n_exact = len(self._find_exact_duplicates(valid))
1225
+ n_near = len(self._find_near_duplicates(valid))
1226
+ lines = [
1227
+ f"Total images: {len(self._records):,}",
1228
+ f"Valid / Corrupt: {len(valid):,} / {corrupt:,}",
1229
+ f"Unique labels: "
1230
+ f"{len(set(r.label for r in valid if r.label)):,}",
1231
+ f"Blurry images: {blurry:,} "
1232
+ f"({blurry/max(len(valid),1)*100:.1f}%)",
1233
+ f"Greyscale-like: {grey_like:,} "
1234
+ f"({grey_like/max(len(valid),1)*100:.1f}%)",
1235
+ f"Exact dupes groups: {n_exact:,}",
1236
+ f"Near dupes groups: {n_near:,}",
1237
+ f"Median H × W: "
1238
+ f"{int(np.median([r.height for r in valid]))} × "
1239
+ f"{int(np.median([r.width for r in valid]))}",
1240
+ ]
1241
+ ax.text(0.04, 0.96, "\n".join(lines),
1242
+ transform=ax.transAxes, va="top", ha="left",
1243
+ fontsize=9.5, color="#1f2328", fontfamily="monospace",
1244
+ bbox=dict(boxstyle="round,pad=0.5", facecolor="#eaeef2",
1245
+ edgecolor="#d0d7de"))
1246
+ ax.set_title("Dataset Overview", color="#1f2328", fontsize=11)
1247
+
1248
+ def _plot_label_dist(self, ax, valid):
1249
+ labels = [r.label for r in valid if r.label]
1250
+ if not labels:
1251
+ ax.text(0.5, 0.5, "No labels provided\n(use label_from_parent=True)",
1252
+ ha="center", va="center", color="#57606a",
1253
+ transform=ax.transAxes, fontsize=9)
1254
+ ax.set_title("Label Distribution", color="#1f2328", fontsize=11)
1255
+ return
1256
+ cnt = Counter(labels).most_common(25)
1257
+ names, counts = zip(*cnt)
1258
+ y = np.arange(len(names))
1259
+ bars = ax.barh(y, counts, color="#58a6ff", alpha=0.85)
1260
+ ax.set_yticks(y)
1261
+ ax.set_yticklabels(names, fontsize=7)
1262
+ ax.set_title("Label Distribution (top 25)", color="#1f2328", fontsize=11)
1263
+ ax.set_xlabel("Count", color="#57606a", fontsize=8)
1264
+ # imbalance ratio
1265
+ if len(counts) > 1:
1266
+ ratio = max(counts) / min(counts)
1267
+ ax.text(0.97, 0.02, f"imbalance ratio: {ratio:.1f}×",
1268
+ transform=ax.transAxes, ha="right", va="bottom",
1269
+ fontsize=7, color="#e3b341")
1270
+
1271
+ def _plot_hist(self, ax, data, title, color, log_x=False):
1272
+ """Histogram with explicit axis metrics for interpretation."""
1273
+ vals = np.array([v for v in data if v is not None and np.isfinite(v)])
1274
+ if log_x:
1275
+ vals = vals[vals > 0]
1276
+ vals = np.log10(vals + 1e-9)
1277
+ x_label = f"log10({title})"
1278
+ else:
1279
+ x_label = title
1280
+
1281
+ if len(vals) == 0:
1282
+ ax.text(0.5, 0.5, "No data", ha="center", va="center")
1283
+ ax.set_title(title)
1284
+ ax.set_xlabel(x_label)
1285
+ ax.set_ylabel("Number of images")
1286
+ return
1287
+
1288
+ ax.hist(vals, bins=30, color=color, alpha=0.85, edgecolor="white")
1289
+ ax.axvline(vals.mean(), color="#1f2328", ls="--", lw=1,
1290
+ label=f"Mean = {vals.mean():.2f}")
1291
+ ax.set_title(title)
1292
+ ax.set_xlabel(x_label)
1293
+ ax.set_ylabel("Number of images")
1294
+ ax.legend(fontsize=7)
1295
+
1296
+ def _plot_channel_histograms(self, ax, valid):
1297
+ means = np.array([r.mean_rgb for r in valid if r.mean_rgb])
1298
+ if not len(means):
1299
+ return
1300
+ colors = ["#ff6b6b", "#51cf66", "#339af0"]
1301
+ names = ["Red", "Green", "Blue"]
1302
+ for i, (c, n) in enumerate(zip(colors, names)):
1303
+ ax.hist(means[:, i], bins=50, color=c, alpha=0.6,
1304
+ label=n, edgecolor="none")
1305
+ ax.set_title("Per-channel Mean Distribution", color="#1f2328", fontsize=9)
1306
+ ax.set_xlabel("Pixel value (0–255)", color="#57606a", fontsize=8)
1307
+ ax.legend(fontsize=7, labelcolor="#1f2328",
1308
+ facecolor="#eaeef2", edgecolor="#d0d7de")
1309
+
1310
+ def _plot_dominant_colours(self, ax, valid):
1311
+ cv2 = _cv2()
1312
+ pool = [r for r in valid if not r.path.startswith("<array")][:60]
1313
+ all_pix = []
1314
+ for rec in pool:
1315
+ img = cv2.imread(str(rec.path))
1316
+ if img is None:
1317
+ continue
1318
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
1319
+ img = cv2.resize(img, (32, 32))
1320
+ all_pix.append(img.reshape(-1, 3))
1321
+ if not all_pix:
1322
+ return
1323
+ pixels = np.vstack(all_pix)
1324
+ try:
1325
+ from sklearn.cluster import MiniBatchKMeans
1326
+ km = MiniBatchKMeans(n_clusters=self.n_colors,
1327
+ random_state=42, n_init=3)
1328
+ if len(pixels) > 50_000:
1329
+ idx = np.random.choice(len(pixels), 50_000, replace=False)
1330
+ pixels = pixels[idx]
1331
+ labels = km.fit_predict(pixels)
1332
+ centers = km.cluster_centers_.astype(np.uint8)
1333
+ counts = np.bincount(labels, minlength=self.n_colors)
1334
+ pct = counts / counts.sum()
1335
+ order = np.argsort(-pct)
1336
+ centers, pct = centers[order], pct[order]
1337
+ hex_cols = [f"#{r:02x}{g:02x}{b:02x}" for r, g, b in centers]
1338
+ ax.barh(np.arange(len(pct)), pct * 100,
1339
+ color=hex_cols, edgecolor="#f6f8fa", height=0.7)
1340
+ ax.set_yticks(np.arange(len(pct)))
1341
+ ax.set_yticklabels(hex_cols, fontsize=8)
1342
+ ax.set_xlabel("% of pixels (sample)", color="#57606a", fontsize=8)
1343
+ ax.set_title("Dominant Colour Palette", color="#1f2328", fontsize=9)
1344
+ except Exception:
1345
+ pass
1346
+
1347
+ def _plot_hsv_hue_wheel(self, ax, valid):
1348
+ cv2 = _cv2()
1349
+ sample = [r for r in valid if not r.path.startswith("<array")][:100]
1350
+ hues = []
1351
+ for rec in sample:
1352
+ img = cv2.imread(str(rec.path))
1353
+ if img is None:
1354
+ continue
1355
+ hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
1356
+ small = cv2.resize(hsv, (32, 32))
1357
+ hues.extend(small[:, :, 0].ravel().tolist())
1358
+ if not hues:
1359
+ return
1360
+ hues = np.array(hues) * 2 # OpenCV hue: 0-180 → 0-360
1361
+ counts, edges = np.histogram(hues, bins=36, range=(0, 360))
1362
+ centres = (edges[:-1] + edges[1:]) / 2
1363
+ colors_h = [_plt().cm.hsv(c / 360) for c in centres]
1364
+ ax.bar(centres, counts, width=10, color=colors_h, alpha=0.9)
1365
+ ax.set_title("Hue Distribution", color="#1f2328", fontsize=9)
1366
+ ax.set_xlabel("Hue (°)", color="#57606a", fontsize=8)
1367
+
1368
+ def _plot_saturation_value(self, ax, valid):
1369
+ sats = [r.mean_hsv[1] if r.mean_hsv else None for r in valid]
1370
+ vals = [r.mean_hsv[2] if r.mean_hsv else None for r in valid]
1371
+ sats = [s for s in sats if s is not None]
1372
+ vals = [v for v in vals if v is not None]
1373
+ ax.scatter(sats, vals, alpha=0.3, s=6, color="#f9c74f", linewidths=0)
1374
+ ax.set_xlabel("Saturation", color="#57606a", fontsize=8)
1375
+ ax.set_ylabel("Value (brightness)", color="#57606a", fontsize=8)
1376
+ ax.set_title("HSV Saturation vs Value", color="#1f2328", fontsize=9)
1377
+
1378
+ def _plot_lab_ab_scatter(self, ax, valid):
1379
+ labs = np.array([r.mean_lab for r in valid if r.mean_lab])
1380
+ if not len(labs):
1381
+ return
1382
+ ax.scatter(labs[:, 1], labs[:, 2], alpha=0.3, s=6,
1383
+ c=labs[:, 0], cmap="viridis", linewidths=0)
1384
+ ax.axhline(0, color="#d0d7de", lw=0.8)
1385
+ ax.axvline(0, color="#d0d7de", lw=0.8)
1386
+ ax.set_xlabel("a* (green–red)", color="#57606a", fontsize=8)
1387
+ ax.set_ylabel("b* (blue–yellow)", color="#57606a", fontsize=8)
1388
+ ax.set_title("Lab Colour Space (a* vs b*)", color="#1f2328", fontsize=9)
1389
+
1390
+ def _plot_color_temp_pie(self, ax, valid):
1391
+ ct = Counter(r.color_temp for r in valid if r.color_temp)
1392
+ if not ct:
1393
+ return
1394
+ colors = {"warm": "#ff6b6b", "neutral": "#57606a", "cool": "#339af0"}
1395
+ lbls = list(ct.keys())
1396
+ vals = [ct[l] for l in lbls]
1397
+ clrs = [colors.get(l, "#white") for l in lbls]
1398
+ wedges, _, autotexts = ax.pie(
1399
+ vals, labels=lbls, colors=clrs,
1400
+ autopct="%1.1f%%", startangle=90,
1401
+ textprops={"color": "#1f2328", "fontsize": 8},
1402
+ )
1403
+ ax.set_title("Colour Temperature", color="#1f2328", fontsize=9)
1404
+
1405
+ def _plot_texture_radar(self, ax, valid):
1406
+ """Draw GLCM radar chart. ax is a plain Axes used only for position;
1407
+ we replace it with a polar subplot in the same grid slot."""
1408
+ feats = ["glcm_contrast", "glcm_homogeneity",
1409
+ "glcm_energy", "glcm_correlation", "glcm_asm"]
1410
+ labels = ["Contrast", "Homogeneity", "Energy", "Correlation", "ASM"]
1411
+ vals = []
1412
+ for f in feats:
1413
+ a = [getattr(r, f) for r in valid if getattr(r, f) is not None]
1414
+ vals.append(np.mean(a) if a else 0.0)
1415
+
1416
+ # Remove the plain axes and replace with a polar one at the same position
1417
+ fig = ax.get_figure()
1418
+ pos = ax.get_position()
1419
+ ax.remove()
1420
+
1421
+ if all(v == 0 for v in vals):
1422
+ ax2 = fig.add_axes(pos)
1423
+ ax2.set_facecolor("#f6f8fa")
1424
+ for sp in ax2.spines.values():
1425
+ sp.set_edgecolor("#d0d7de")
1426
+ ax2.text(0.5, 0.5, "GLCM not computed\n(install scikit-image)",
1427
+ ha="center", va="center", color="#57606a",
1428
+ transform=ax2.transAxes, fontsize=9)
1429
+ ax2.set_title("Texture Features (GLCM)", color="#1f2328", fontsize=9)
1430
+ ax2.set_xticks([]); ax2.set_yticks([])
1431
+ return
1432
+
1433
+ ax_polar = fig.add_axes(pos, projection="polar")
1434
+ ax_polar.set_facecolor("#f6f8fa")
1435
+
1436
+ # normalise
1437
+ mx = max(vals) or 1.0
1438
+ norm_vals = [v / mx for v in vals]
1439
+ norm_vals += norm_vals[:1]
1440
+
1441
+ angles = np.linspace(0, 2 * np.pi, len(labels), endpoint=False).tolist()
1442
+ angles += angles[:1]
1443
+
1444
+ ax_polar.set_theta_offset(np.pi / 2)
1445
+ ax_polar.set_theta_direction(-1)
1446
+ ax_polar.plot(angles, norm_vals, color="#58a6ff", lw=2)
1447
+ ax_polar.fill(angles, norm_vals, color="#58a6ff", alpha=0.25)
1448
+ ax_polar.set_xticks(angles[:-1])
1449
+ ax_polar.set_xticklabels(labels, color="#1f2328", fontsize=7)
1450
+ ax_polar.set_yticks([0.25, 0.5, 0.75, 1.0])
1451
+ ax_polar.set_yticklabels(["25%", "50%", "75%", "100%"],
1452
+ color="#57606a", fontsize=6)
1453
+ ax_polar.spines["polar"].set_edgecolor("#d0d7de")
1454
+ ax_polar.tick_params(colors="#57606a", labelsize=7)
1455
+ ax_polar.set_title("Texture Features (GLCM, normalised)",
1456
+ color="#1f2328", fontsize=9, pad=12)
1457
+
1458
+ def _plot_frequency_bands(self, ax, valid):
1459
+ low = [r.freq_low for r in valid if r.freq_low is not None]
1460
+ mid = [r.freq_mid for r in valid if r.freq_mid is not None]
1461
+ high = [r.freq_high for r in valid if r.freq_high is not None]
1462
+ if not low:
1463
+ ax.text(0.5, 0.5, "Frequency analysis\nnot computed",
1464
+ ha="center", va="center", color="#57606a",
1465
+ transform=ax.transAxes)
1466
+ ax.set_title("FFT Frequency Bands", color="#1f2328", fontsize=9)
1467
+ return
1468
+ x = np.arange(3)
1469
+ means = [np.mean(low), np.mean(mid), np.mean(high)]
1470
+ stds = [np.std(low), np.std(mid), np.std(high)]
1471
+ colors = ["#58a6ff", "#3fb950", "#e3b341"]
1472
+ ax.bar(x, means, yerr=stds, color=colors, alpha=0.85,
1473
+ edgecolor="none", capsize=4,
1474
+ error_kw=dict(ecolor="#1f2328", elinewidth=1))
1475
+ ax.set_xticks(x)
1476
+ ax.set_xticklabels(["Low\n(<10%)", "Mid\n(10–40%)", "High\n(>40%)"],
1477
+ color="#57606a", fontsize=8)
1478
+ ax.set_title("FFT Frequency Band Energy\n(mean ± std)",
1479
+ color="#1f2328", fontsize=9)
1480
+ ax.set_ylabel("Fraction of total energy", color="#57606a", fontsize=8)
1481
+
1482
+ def _plot_brightness_sharpness_scatter(self, ax, valid):
1483
+ b = [r.brightness for r in valid]
1484
+ s = [r.sharpness for r in valid]
1485
+ c = [r.contrast for r in valid]
1486
+ sc = ax.scatter(b, s, c=c, cmap="plasma", s=6,
1487
+ alpha=0.4, linewidths=0)
1488
+ ax.set_yscale("log")
1489
+ ax.set_xlabel("Brightness", color="#57606a", fontsize=8)
1490
+ ax.set_ylabel("Sharpness (log)", color="#57606a", fontsize=8)
1491
+ ax.set_title("Brightness vs Sharpness\n(colour = contrast)",
1492
+ color="#1f2328", fontsize=9)
1493
+ _plt().colorbar(sc, ax=ax, fraction=0.046, pad=0.04,
1494
+ label="Contrast").ax.tick_params(
1495
+ labelcolor="#57606a", labelsize=6)
1496
+
1497
+ def _plot_exposure_scatter(self, ax, valid):
1498
+ over = [r.overexposed_frac * 100 for r in valid]
1499
+ under = [r.underexposed_frac * 100 for r in valid]
1500
+ ax.scatter(under, over, alpha=0.3, s=6,
1501
+ color="#79c0ff", linewidths=0)
1502
+ ax.set_xlabel("Underexposed pixels (%)", color="#57606a", fontsize=8)
1503
+ ax.set_ylabel("Overexposed pixels (%)", color="#57606a", fontsize=8)
1504
+ ax.set_title("Exposure Map", color="#1f2328", fontsize=9)
1505
+
1506
+ def _plot_blurry_breakdown(self, ax, valid):
1507
+ sharp = sum(not r.is_blurry for r in valid)
1508
+ blurry = sum(r.is_blurry for r in valid)
1509
+ ax.bar(["Sharp", "Blurry"], [sharp, blurry],
1510
+ color=["#3fb950", "#f78166"], alpha=0.85, edgecolor="none")
1511
+ ax.set_title(f"Blurry Detection\n(threshold={self.blur_threshold})",
1512
+ color="#1f2328", fontsize=9)
1513
+ for i, v in enumerate([sharp, blurry]):
1514
+ ax.text(i, v + max(sharp, blurry) * 0.02, str(v),
1515
+ ha="center", va="bottom", color="#1f2328", fontsize=9)
1516
+
1517
+ def _plot_format_dist(self, ax, valid):
1518
+ cnt = Counter(r.file_ext for r in valid)
1519
+ if not cnt:
1520
+ return
1521
+ names, counts = zip(*cnt.most_common())
1522
+ colors = ["#58a6ff", "#3fb950", "#e3b341", "#f78166",
1523
+ "#d2a8ff", "#ffa657", "#79c0ff", "#56d364"]
1524
+ ax.bar(names, counts,
1525
+ color=colors[:len(names)], alpha=0.85, edgecolor="none")
1526
+ ax.set_title("File Format Distribution", color="#1f2328", fontsize=9)
1527
+ ax.set_xlabel("Extension", color="#57606a", fontsize=8)
1528
+ ax.set_ylabel("Count", color="#57606a", fontsize=8)
1529
+
1530
+ # ─────────────────────────────────────────────────────────────────────
1531
+ # Helpers
1532
+ # ─────────────────────────────────────────────────────────────────────
1533
+
1534
+ def _resolve_paths(self, source, recursive) -> List[Path]:
1535
+ if isinstance(source, (list, tuple)):
1536
+ return [Path(p) for p in source]
1537
+ source = Path(source)
1538
+ if source.is_file():
1539
+ return [source]
1540
+ pattern = "**/*" if recursive else "*"
1541
+ return sorted(
1542
+ p for p in source.glob(pattern)
1543
+ if p.is_file() and p.suffix.lower() in self.SUPPORTED_EXTS
1544
+ )
1545
+
1546
+ @staticmethod
1547
+ def _to_uint8_rgb(arr: np.ndarray) -> np.ndarray:
1548
+ if arr.dtype in (np.float32, np.float64):
1549
+ arr = (np.clip(arr, 0, 1) * 255).astype(np.uint8)
1550
+ if arr.ndim == 2:
1551
+ arr = np.stack([arr, arr, arr], axis=-1)
1552
+ elif arr.shape[2] == 4:
1553
+ arr = arr[:, :, :3]
1554
+ return arr.astype(np.uint8)
1555
+
1556
+ def _log(self, msg: str) -> None:
1557
+ if self.verbose:
1558
+ print(f"[viseda] {msg}")
1559
+
1560
+ def _check_loaded(self):
1561
+ if not self._loaded:
1562
+ raise RuntimeError("Call .load() or .load_arrays() before analysing.")
1563
+
1564
+ @staticmethod
1565
+ def _finalise(fig, save_path, dpi):
1566
+ plt = _plt()
1567
+ if save_path:
1568
+ fig.savefig(save_path, dpi=dpi, bbox_inches="tight",
1569
+ facecolor=fig.get_facecolor())
1570
+ else:
1571
+ plt.show()
1572
+ plt.close(fig)
1573
+
1574
+
1575
+ # ═════════════════════════════════════════════════════════════════════════════
1576
+ # Utilities
1577
+ # ═════════════════════════════════════════════════════════════════════════════
1578
+
1579
+ def _stat_dict(arr) -> Dict[str, float]:
1580
+ arr = np.asarray([x for x in arr if x is not None and np.isfinite(x)])
1581
+ if len(arr) == 0:
1582
+ return {}
1583
+ return {
1584
+ "min": round(float(np.min(arr)), 4),
1585
+ "max": round(float(np.max(arr)), 4),
1586
+ "mean": round(float(np.mean(arr)), 4),
1587
+ "median": round(float(np.median(arr)), 4),
1588
+ "std": round(float(np.std(arr)), 4),
1589
+ "p25": round(float(np.percentile(arr, 25)), 4),
1590
+ "p75": round(float(np.percentile(arr, 75)), 4),
1591
+ }
1592
+
1593
+
1594
+ def _resize_display(img: np.ndarray, max_side: int) -> np.ndarray:
1595
+ cv2 = _cv2()
1596
+ h, w = img.shape[:2]
1597
+ scale = min(max_side / h, max_side / w, 1.0)
1598
+ if scale < 1.0:
1599
+ img = cv2.resize(img, (int(w * scale), int(h * scale)),
1600
+ interpolation=cv2.INTER_AREA)
1601
+ return img
1602
+
1603
+
1604
+ # ═════════════════════════════════════════════════════════════════════════════
1605
+ # HTML Report
1606
+ # ═════════════════════════════════════════════════════════════════════════════
1607
+
1608
+ def _generate_html_report(summary: Dict[str, Any], output_path: str) -> None:
1609
+ inv = summary.get("inventory", {})
1610
+ sp = summary.get("spatial", {})
1611
+ px = summary.get("pixel_stats", {})
1612
+ qu = summary.get("quality", {})
1613
+ co = summary.get("colour", {})
1614
+ tx = summary.get("texture", {})
1615
+ fr = summary.get("frequency", {})
1616
+ du = summary.get("duplicates",{})
1617
+ lb = summary.get("labels", {})
1618
+ nm = summary.get("normalization_stats", {})
1619
+
1620
+ def card(title, stats):
1621
+ if not stats:
1622
+ return ""
1623
+ rows = "".join(
1624
+ f'<div class="stat"><span>{k}</span>'
1625
+ f'<span class="val">{_fmt(v)}</span></div>'
1626
+ for k, v in stats.items()
1627
+ )
1628
+ return f'<div class="card"><h3>{title}</h3>{rows}</div>'
1629
+
1630
+ def badge(text, cls="blue"):
1631
+ return f'<span class="badge badge-{cls}">{text}</span>'
1632
+
1633
+ def bar_chart(title, dist, span=1):
1634
+ if not dist:
1635
+ return ""
1636
+ total = sum(dist.values()) or 1
1637
+ mx = max(dist.values()) or 1
1638
+ rows = ""
1639
+ for lbl, cnt in sorted(dist.items(), key=lambda x: -x[1])[:25]:
1640
+ pct = cnt / mx * 100
1641
+ rows += (
1642
+ f'<div class="bar-row">'
1643
+ f'<span class="bar-label" title="{lbl}">{lbl}</span>'
1644
+ f'<div class="bar">'
1645
+ f'<div class="bar-fill" style="width:{pct:.1f}%"></div></div>'
1646
+ f'<span class="bar-count">{cnt:,}</span></div>'
1647
+ )
1648
+ span_style = f'grid-column: span {span};' if span > 1 else ''
1649
+ return (f'<div class="card" style="{span_style}">'
1650
+ f'<h3>{title}</h3><div class="bar-wrap">{rows}</div></div>')
1651
+
1652
+ badges_html = (
1653
+ badge(f"{inv.get('total', 0):,} images") +
1654
+ badge(f"{inv.get('valid', 0):,} valid", "green") +
1655
+ (badge(f"{inv.get('corrupt', 0):,} corrupt", "red")
1656
+ if inv.get("corrupt") else "") +
1657
+ badge(f"{du.get('n_exact_duplicate_groups', 0)} exact dupes", "yellow") +
1658
+ badge(f"{du.get('n_near_duplicate_groups', 0)} near dupes", "yellow")
1659
+ )
1660
+
1661
+ norm_html = ""
1662
+ if nm:
1663
+ m = [f"{v:.4f}" for v in nm.get("mean", [])]
1664
+ s = [f"{v:.4f}" for v in nm.get("std", [])]
1665
+ norm_html = f"""
1666
+ <h2>📐 Normalisation Stats (for torchvision / transforms)</h2>
1667
+ <div class="card"><pre style="color:#58a6ff;font-size:0.85rem;">
1668
+ transforms.Normalize(
1669
+ mean={m},
1670
+ std ={s}
1671
+ )</pre></div>"""
1672
+
1673
+ corrupt_html = ""
1674
+ if inv.get("corrupt_paths"):
1675
+ items = "".join(f"<li>{p}</li>"
1676
+ for p in inv["corrupt_paths"][:50])
1677
+ corrupt_html = (f'<h2>⚠️ Corrupt Files</h2>'
1678
+ f'<div class="card corrupt"><ul>{items}</ul></div>')
1679
+
1680
+ # Pre-build dicts that can't go inside f-string {{ }} literals
1681
+ blurry_card = card("Blurry Summary", {
1682
+ "Blurry count": qu.get("blurry_count"),
1683
+ "Blurry fraction": qu.get("blurry_fraction"),
1684
+ })
1685
+ greyscale_card = card("Greyscale-like", {
1686
+ "Count": co.get("grayscale_like_count"),
1687
+ "Fraction": co.get("grayscale_like_fraction"),
1688
+ })
1689
+ duplicate_card = card("Duplicate Summary", {
1690
+ "Exact duplicate groups": du.get("n_exact_duplicate_groups"),
1691
+ "Near duplicate groups": du.get("n_near_duplicate_groups"),
1692
+ })
1693
+ class_imb_card = card("Class Imbalance", {
1694
+ "Imbalance ratio (max/min)": lb.get("class_imbalance_ratio"),
1695
+ })
1696
+ counts_card = card("Counts", {
1697
+ "Total": inv.get("total"),
1698
+ "Valid": inv.get("valid"),
1699
+ "Corrupt": inv.get("corrupt"),
1700
+ })
1701
+ mean_rgb_card = card("Dataset Mean RGB (0-255)",
1702
+ dict(zip(["R mean", "G mean", "B mean"],
1703
+ [round(v, 3) for v in px.get("dataset_mean_rgb", [])])))
1704
+ std_rgb_card = card("Dataset Pixel Std RGB",
1705
+ dict(zip(["R std", "G std", "B std"],
1706
+ [round(v, 3) for v in px.get("dataset_pixel_std_rgb", [])])))
1707
+
1708
+ html = f"""<!DOCTYPE html>
1709
+ <html lang="en"><head><meta charset="UTF-8"/>
1710
+ <meta name="viewport" content="width=device-width,initial-scale=1"/>
1711
+ <title>VisEDA Report</title>
1712
+ <style>
1713
+ :root{{--bg:white;--surface:#f6f8fa;--border:#d0d7de;--text:#1f2328;
1714
+ --muted:#57606a;--accent:#58a6ff;--green:#3fb950;--red:#f78166;
1715
+ --yellow:#e3b341;}}
1716
+ *{{box-sizing:border-box;margin:0;padding:0}}
1717
+ body{{background:var(--bg);color:var(--text);
1718
+ font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;
1719
+ padding:2rem;}}
1720
+ h1{{font-size:1.9rem;margin-bottom:.25rem}}
1721
+ h2{{font-size:1.05rem;color:var(--accent);margin:1.8rem 0 .6rem;}}
1722
+ h3{{font-size:.78rem;color:var(--muted);text-transform:uppercase;
1723
+ letter-spacing:.05em;margin-bottom:.5rem}}
1724
+ .sub{{color:var(--muted);font-size:.85rem;margin-bottom:1.5rem}}
1725
+ .grid{{display:grid;grid-template-columns:repeat(auto-fill,minmax(240px,1fr));
1726
+ gap:.9rem}}
1727
+ .card{{background:var(--surface);border:1px solid var(--border);
1728
+ border-radius:8px;padding:1rem}}
1729
+ .stat{{display:flex;justify-content:space-between;font-size:.82rem;
1730
+ padding:.18rem 0;border-bottom:1px solid var(--border)}}
1731
+ .stat:last-child{{border-bottom:none}}
1732
+ .val{{color:var(--accent);font-variant-numeric:tabular-nums}}
1733
+ .badge{{display:inline-block;padding:.15rem .5rem;border-radius:12px;
1734
+ font-size:.72rem;font-weight:600;margin:.15rem}}
1735
+ .badge-blue{{background:rgba(88,166,255,.15);color:var(--accent)}}
1736
+ .badge-green{{background:rgba(63,185,80,.15);color:var(--green)}}
1737
+ .badge-red{{background:rgba(247,129,102,.15);color:var(--red)}}
1738
+ .badge-yellow{{background:rgba(227,179,65,.15);color:var(--yellow)}}
1739
+ .bar-wrap{{margin-top:.4rem}}
1740
+ .bar-row{{display:flex;align-items:center;gap:.4rem;margin:.18rem 0;
1741
+ font-size:.76rem}}
1742
+ .bar-label{{width:120px;overflow:hidden;text-overflow:ellipsis;
1743
+ white-space:nowrap;color:var(--muted)}}
1744
+ .bar{{flex:1;background:var(--border);border-radius:3px;height:9px}}
1745
+ .bar-fill{{height:100%;border-radius:3px;background:var(--accent)}}
1746
+ .bar-count{{width:55px;text-align:right;color:var(--accent)}}
1747
+ .corrupt{{max-height:200px;overflow-y:auto;font-size:.78rem;
1748
+ color:var(--red)}}
1749
+ pre{{background:#eaeef2;padding:.8rem;border-radius:6px;
1750
+ overflow-x:auto;font-size:.82rem}}
1751
+ footer{{margin-top:3rem;color:var(--muted);font-size:.72rem;
1752
+ border-top:1px solid var(--border);padding-top:1rem}}
1753
+ </style></head><body>
1754
+ <h1>🔬 VisEDA — Image EDA Report</h1>
1755
+ <p class="sub">Generated by <strong>VisEDA</strong></p>
1756
+ <p style="margin-bottom:1rem">{badges_html}</p>
1757
+
1758
+ <h2>📦 Inventory</h2>
1759
+ <div class="grid">
1760
+ {counts_card}
1761
+ {bar_chart("Format Distribution", inv.get("format_distribution", {}))}
1762
+ {bar_chart("Colour Mode", inv.get("colour_mode_distribution", {}))}
1763
+ </div>
1764
+
1765
+ <h2>📐 Spatial</h2>
1766
+ <div class="grid">
1767
+ {card("Height (px)", sp.get("height", {}))}
1768
+ {card("Width (px)", sp.get("width", {}))}
1769
+ {card("Aspect Ratio", sp.get("aspect_ratio", {}))}
1770
+ {card("Megapixels", sp.get("megapixels", {}))}
1771
+ {card("File Size (KB)", sp.get("file_size_kb", {}))}
1772
+ {bar_chart("Orientation", sp.get("orientation_distribution", {}))}
1773
+ </div>
1774
+
1775
+ <h2>🎨 Pixel Statistics</h2>
1776
+ <div class="grid">
1777
+ {mean_rgb_card}
1778
+ {std_rgb_card}
1779
+ </div>
1780
+
1781
+ {norm_html}
1782
+
1783
+ <h2>🔍 Quality Metrics</h2>
1784
+ <div class="grid">
1785
+ {card("Brightness", qu.get("brightness", {}))}
1786
+ {card("Contrast", qu.get("contrast", {}))}
1787
+ {card("Sharpness", qu.get("sharpness", {}))}
1788
+ {card("Noise Estimate", qu.get("noise_estimate", {}))}
1789
+ {card("Pixel Entropy", qu.get("entropy", {}))}
1790
+ {card("Compression Score", qu.get("compression_score",{}))}
1791
+ {card("Overexposed Frac", qu.get("overexposed_frac", {}))}
1792
+ {card("Underexposed Frac", qu.get("underexposed_frac",{}))}
1793
+ {blurry_card}
1794
+ </div>
1795
+
1796
+ <h2>🌈 Colour</h2>
1797
+ <div class="grid">
1798
+ {card("Saturation (HSV-S)", co.get("saturation", {}))}
1799
+ {bar_chart("Colour Temperature", co.get("colour_temp_distribution", {}))}
1800
+ {greyscale_card}
1801
+ </div>
1802
+
1803
+ <h2>🧱 Texture (GLCM)</h2>
1804
+ <div class="grid">
1805
+ {"".join(card(k.replace("_"," ").title(), v) for k, v in tx.items())}
1806
+ </div>
1807
+
1808
+ <h2>〰️ Frequency (FFT)</h2>
1809
+ <div class="grid">
1810
+ {"".join(card(k.replace("_"," ").title(), v) for k, v in fr.items())}
1811
+ </div>
1812
+
1813
+ <h2>🔁 Duplicates</h2>
1814
+ <div class="grid">
1815
+ {duplicate_card}
1816
+ </div>
1817
+
1818
+ <h2>🏷️ Labels</h2>
1819
+ <div class="grid">
1820
+ {bar_chart("Label Distribution",
1821
+ lb.get("label_distribution") or {}, span=2)}
1822
+ {class_imb_card}
1823
+ </div>
1824
+
1825
+ {corrupt_html}
1826
+
1827
+ <footer>Generated by VisEDA — Visual Exploratory Data Analysis</footer>
1828
+ </body></html>"""
1829
+
1830
+ Path(output_path).write_text(html, encoding="utf-8")
1831
+
1832
+
1833
+ def _fmt(v) -> str:
1834
+ if v is None:
1835
+ return "N/A"
1836
+ if isinstance(v, float):
1837
+ return f"{v:,.4f}"
1838
+ if isinstance(v, int):
1839
+ return f"{v:,}"
1840
+ return str(v)