viseda 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- viseda/__init__.py +20 -0
- viseda/cli.py +144 -0
- viseda/core/__init__.py +3 -0
- viseda/core/base.py +69 -0
- viseda/hyperspectral/__init__.py +1 -0
- viseda/hyperspectral/eda.py +1849 -0
- viseda/image/__init__.py +1 -0
- viseda/image/eda.py +1840 -0
- viseda/pointcloud/__init__.py +3 -0
- viseda/pointcloud/eda.py +1167 -0
- viseda/report/__init__.py +3 -0
- viseda/report/html_report.py +213 -0
- viseda/text/__init__.py +3 -0
- viseda/text/eda.py +1613 -0
- viseda/utils/__init__.py +27 -0
- viseda/utils/helpers.py +120 -0
- viseda/video/__init__.py +3 -0
- viseda/video/eda.py +587 -0
- viseda-1.0.0.dist-info/METADATA +266 -0
- viseda-1.0.0.dist-info/RECORD +24 -0
- viseda-1.0.0.dist-info/WHEEL +5 -0
- viseda-1.0.0.dist-info/entry_points.txt +2 -0
- viseda-1.0.0.dist-info/licenses/LICENSE +21 -0
- viseda-1.0.0.dist-info/top_level.txt +1 -0
viseda/image/eda.py
ADDED
|
@@ -0,0 +1,1840 @@
|
|
|
1
|
+
"""
|
|
2
|
+
viseda.image.eda
|
|
3
|
+
----------------
|
|
4
|
+
Comprehensive EDA for image datasets.
|
|
5
|
+
|
|
6
|
+
Analyses
|
|
7
|
+
~~~~~~~~
|
|
8
|
+
INVENTORY
|
|
9
|
+
- Total / valid / corrupt count
|
|
10
|
+
- File format & extension distribution
|
|
11
|
+
- File size distribution (KB)
|
|
12
|
+
- Colour-mode distribution (RGB / grayscale / RGBA)
|
|
13
|
+
- Bit-depth distribution
|
|
14
|
+
|
|
15
|
+
SPATIAL
|
|
16
|
+
- Height, width, aspect-ratio distributions
|
|
17
|
+
- Resolution (megapixels) distribution
|
|
18
|
+
- Portrait / landscape / square breakdown
|
|
19
|
+
- Spatial resolution consistency check
|
|
20
|
+
|
|
21
|
+
PIXEL STATISTICS (per image, then aggregated across dataset)
|
|
22
|
+
- Mean, std, min, max per channel
|
|
23
|
+
- Per-channel histograms aggregated across the dataset
|
|
24
|
+
- Global dataset-level pixel mean and std ("dataset statistics" for
|
|
25
|
+
normalisation — the same numbers used in torchvision transforms)
|
|
26
|
+
|
|
27
|
+
COLOUR ANALYSIS
|
|
28
|
+
- RGB, HSV and Lab colour space distributions
|
|
29
|
+
- Dominant colour palette extraction (K-Means across sample)
|
|
30
|
+
- Colour temperature estimate (warm / neutral / cool)
|
|
31
|
+
- Colour cast detection (channel imbalance)
|
|
32
|
+
- Greyscale-like detection (low colour saturation)
|
|
33
|
+
|
|
34
|
+
QUALITY METRICS (per image, then aggregated)
|
|
35
|
+
- Brightness (mean luminance)
|
|
36
|
+
- Contrast (std of luminance)
|
|
37
|
+
- Sharpness (Laplacian variance)
|
|
38
|
+
- Noise estimate (high-freq energy via Laplacian on smooth image)
|
|
39
|
+
- Exposure (over- / under-exposed pixel fraction)
|
|
40
|
+
- Blurriness flag (sharpness below threshold)
|
|
41
|
+
- JPEG compression artefact score (blockiness)
|
|
42
|
+
|
|
43
|
+
TEXTURE & FREQUENCY
|
|
44
|
+
- GLCM texture features: contrast, dissimilarity, homogeneity,
|
|
45
|
+
energy, correlation, ASM (per image → aggregated)
|
|
46
|
+
- FFT frequency energy distribution (low / mid / high freq ratio)
|
|
47
|
+
|
|
48
|
+
DUPLICATE DETECTION
|
|
49
|
+
- Perceptual hash (pHash, 64-bit DCT)
|
|
50
|
+
- Average hash (aHash)
|
|
51
|
+
- Near-duplicate grouping (Hamming distance ≤ threshold)
|
|
52
|
+
- Exact-duplicate detection (MD5)
|
|
53
|
+
|
|
54
|
+
DATASET-LEVEL
|
|
55
|
+
- Class / label distribution (from folder names or dict)
|
|
56
|
+
- Per-class pixel statistics
|
|
57
|
+
- Class imbalance ratio
|
|
58
|
+
- Outlier image detection (images far from dataset mean embedding)
|
|
59
|
+
|
|
60
|
+
VISUALISATIONS
|
|
61
|
+
- Full EDA dashboard (light theme, 6×4 grid)
|
|
62
|
+
- Sample image grid
|
|
63
|
+
- Per-class sample grid
|
|
64
|
+
- Channel correlation matrix
|
|
65
|
+
- Pixel-value heatmap (average image across dataset)
|
|
66
|
+
- UMAP / t-SNE embedding of image features (optional)
|
|
67
|
+
- Duplicate group viewer
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
from __future__ import annotations
|
|
71
|
+
|
|
72
|
+
import hashlib
|
|
73
|
+
import warnings
|
|
74
|
+
from collections import Counter, defaultdict
|
|
75
|
+
from pathlib import Path
|
|
76
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
|
|
77
|
+
|
|
78
|
+
import numpy as np
|
|
79
|
+
|
|
80
|
+
# ── lazy heavy imports ────────────────────────────────────────────────────
|
|
81
|
+
def _cv2():
|
|
82
|
+
import cv2; return cv2
|
|
83
|
+
|
|
84
|
+
def _plt():
|
|
85
|
+
import matplotlib.pyplot as plt; return plt
|
|
86
|
+
|
|
87
|
+
def _mpl():
|
|
88
|
+
import matplotlib as mpl; return mpl
|
|
89
|
+
|
|
90
|
+
def _ski_feature():
|
|
91
|
+
from skimage.feature import graycomatrix, graycoprops
|
|
92
|
+
return graycomatrix, graycoprops
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
# ═════════════════════════════════════════════════════════════════════════════
|
|
96
|
+
# Per-image record
|
|
97
|
+
# ═════════════════════════════════════════════════════════════════════════════
|
|
98
|
+
|
|
99
|
+
class ImageRecord:
|
|
100
|
+
"""All per-image statistics stored in one lightweight object."""
|
|
101
|
+
|
|
102
|
+
__slots__ = (
|
|
103
|
+
# identity
|
|
104
|
+
"path", "label", "file_ext", "file_size_kb",
|
|
105
|
+
# spatial
|
|
106
|
+
"height", "width", "channels", "dtype",
|
|
107
|
+
"aspect_ratio", "megapixels",
|
|
108
|
+
# pixel stats
|
|
109
|
+
"mean_rgb", "std_rgb", "min_rgb", "max_rgb",
|
|
110
|
+
# quality
|
|
111
|
+
"brightness", "contrast", "sharpness",
|
|
112
|
+
"noise_estimate",
|
|
113
|
+
"overexposed_frac", "underexposed_frac",
|
|
114
|
+
"is_blurry",
|
|
115
|
+
"compression_score",
|
|
116
|
+
# colour
|
|
117
|
+
"mean_hsv", "mean_lab",
|
|
118
|
+
"saturation_mean",
|
|
119
|
+
"color_temp", # "warm" | "neutral" | "cool"
|
|
120
|
+
"is_grayscale_like",
|
|
121
|
+
# texture
|
|
122
|
+
"glcm_contrast", "glcm_dissimilarity", "glcm_homogeneity",
|
|
123
|
+
"glcm_energy", "glcm_correlation", "glcm_asm",
|
|
124
|
+
# frequency
|
|
125
|
+
"freq_low", "freq_mid", "freq_high",
|
|
126
|
+
# entropy
|
|
127
|
+
"entropy_val",
|
|
128
|
+
# hashes
|
|
129
|
+
"phash", "ahash", "md5",
|
|
130
|
+
# status
|
|
131
|
+
"is_corrupt",
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
def __init__(self):
|
|
135
|
+
for s in self.__slots__:
|
|
136
|
+
setattr(self, s, None)
|
|
137
|
+
self.is_corrupt = False
|
|
138
|
+
self.is_blurry = False
|
|
139
|
+
self.is_grayscale_like = False
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# ═════════════════════════════════════════════════════════════════════════════
|
|
143
|
+
# Main class
|
|
144
|
+
# ═════════════════════════════════════════════════════════════════════════════
|
|
145
|
+
|
|
146
|
+
class ImageEDA:
|
|
147
|
+
"""
|
|
148
|
+
Comprehensive exploratory data analysis for image datasets.
|
|
149
|
+
|
|
150
|
+
Parameters
|
|
151
|
+
----------
|
|
152
|
+
verbose : bool
|
|
153
|
+
Print progress messages.
|
|
154
|
+
max_images : int | None
|
|
155
|
+
Analyse at most *max_images* (useful for quick previews).
|
|
156
|
+
n_colors : int
|
|
157
|
+
Dominant-colour clusters to extract (default 8).
|
|
158
|
+
phash_threshold : int
|
|
159
|
+
Hamming distance threshold for near-duplicate detection (default 10).
|
|
160
|
+
blur_threshold : float
|
|
161
|
+
Laplacian variance below this flags an image as blurry (default 50).
|
|
162
|
+
compute_glcm : bool
|
|
163
|
+
Compute GLCM texture features — requires scikit-image (default True).
|
|
164
|
+
compute_freq : bool
|
|
165
|
+
Compute FFT frequency band energies (default True).
|
|
166
|
+
|
|
167
|
+
Examples
|
|
168
|
+
--------
|
|
169
|
+
>>> from viseda import ImageEDA
|
|
170
|
+
>>> eda = ImageEDA()
|
|
171
|
+
>>> eda.load("path/to/dataset/", label_from_parent=True)
|
|
172
|
+
>>> print(eda.summary())
|
|
173
|
+
>>> eda.plot()
|
|
174
|
+
>>> eda.plot_samples()
|
|
175
|
+
>>> eda.plot_average_image()
|
|
176
|
+
>>> eda.report("report.html")
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
SUPPORTED_EXTS = {".jpg", ".jpeg", ".png", ".bmp",
|
|
180
|
+
".tif", ".tiff", ".webp", ".gif"}
|
|
181
|
+
|
|
182
|
+
def __init__(
|
|
183
|
+
self,
|
|
184
|
+
verbose: bool = True,
|
|
185
|
+
max_images: Optional[int] = None,
|
|
186
|
+
n_colors: int = 8,
|
|
187
|
+
phash_threshold: int = 10,
|
|
188
|
+
blur_threshold: float = 50.0,
|
|
189
|
+
compute_glcm: bool = True,
|
|
190
|
+
compute_freq: bool = True,
|
|
191
|
+
):
|
|
192
|
+
self.verbose = verbose
|
|
193
|
+
self.max_images = max_images
|
|
194
|
+
self.n_colors = n_colors
|
|
195
|
+
self.phash_threshold = phash_threshold
|
|
196
|
+
self.blur_threshold = blur_threshold
|
|
197
|
+
self.compute_glcm = compute_glcm
|
|
198
|
+
self.compute_freq = compute_freq
|
|
199
|
+
|
|
200
|
+
self._records: List[ImageRecord] = []
|
|
201
|
+
self._labels_map: Dict[str, str] = {}
|
|
202
|
+
self._loaded = False
|
|
203
|
+
self._results: Dict[str, Any] = {}
|
|
204
|
+
|
|
205
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
206
|
+
# Loading
|
|
207
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
208
|
+
|
|
209
|
+
def load(
|
|
210
|
+
self,
|
|
211
|
+
source: Union[str, Path, List],
|
|
212
|
+
labels: Optional[Dict[str, str]] = None,
|
|
213
|
+
label_from_parent: bool = False,
|
|
214
|
+
recursive: bool = True,
|
|
215
|
+
) -> "ImageEDA":
|
|
216
|
+
"""
|
|
217
|
+
Load images from a directory, file, or list of paths.
|
|
218
|
+
|
|
219
|
+
Parameters
|
|
220
|
+
----------
|
|
221
|
+
source
|
|
222
|
+
Directory path, single image path, or list of paths.
|
|
223
|
+
labels
|
|
224
|
+
``{path: label}`` mapping.
|
|
225
|
+
label_from_parent
|
|
226
|
+
Infer label from the parent folder name
|
|
227
|
+
(e.g. ``dataset/cats/img.jpg`` → label ``"cats"``).
|
|
228
|
+
recursive
|
|
229
|
+
Recurse into sub-directories (default True).
|
|
230
|
+
"""
|
|
231
|
+
paths = self._resolve_paths(source, recursive)
|
|
232
|
+
if self.max_images:
|
|
233
|
+
paths = paths[: self.max_images]
|
|
234
|
+
|
|
235
|
+
if labels:
|
|
236
|
+
self._labels_map = {str(Path(k).resolve()): v
|
|
237
|
+
for k, v in labels.items()}
|
|
238
|
+
|
|
239
|
+
self._log(f"Found {len(paths)} images — computing statistics …")
|
|
240
|
+
self._records = []
|
|
241
|
+
|
|
242
|
+
for i, p in enumerate(paths):
|
|
243
|
+
if self.verbose and i % max(1, len(paths) // 20) == 0:
|
|
244
|
+
self._log(f" [{i:>{len(str(len(paths)))}}/{len(paths)}] {p.name}")
|
|
245
|
+
rec = self._analyse_single(p, label_from_parent)
|
|
246
|
+
self._records.append(rec)
|
|
247
|
+
|
|
248
|
+
self._loaded = True
|
|
249
|
+
n_corrupt = sum(r.is_corrupt for r in self._records)
|
|
250
|
+
self._log(
|
|
251
|
+
f"Done. {len(self._records):,} images loaded "
|
|
252
|
+
f"({n_corrupt} corrupt)."
|
|
253
|
+
)
|
|
254
|
+
return self
|
|
255
|
+
|
|
256
|
+
def load_arrays(
|
|
257
|
+
self,
|
|
258
|
+
arrays: List[np.ndarray],
|
|
259
|
+
labels: Optional[List[str]] = None,
|
|
260
|
+
) -> "ImageEDA":
|
|
261
|
+
"""Load images directly as NumPy arrays (HxWx3 uint8 or float)."""
|
|
262
|
+
self._log(f"Loading {len(arrays)} arrays …")
|
|
263
|
+
self._records = []
|
|
264
|
+
for i, arr in enumerate(arrays):
|
|
265
|
+
rec = ImageRecord()
|
|
266
|
+
rec.path = f"<array_{i}>"
|
|
267
|
+
rec.label = labels[i] if labels and i < len(labels) else None
|
|
268
|
+
rec.file_ext = "array"
|
|
269
|
+
try:
|
|
270
|
+
self._fill_stats(rec, self._to_uint8_rgb(arr))
|
|
271
|
+
except Exception as e:
|
|
272
|
+
rec.is_corrupt = True
|
|
273
|
+
self._log(f" ✗ array_{i}: {e}")
|
|
274
|
+
self._records.append(rec)
|
|
275
|
+
self._loaded = True
|
|
276
|
+
return self
|
|
277
|
+
|
|
278
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
279
|
+
# Summary
|
|
280
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
281
|
+
|
|
282
|
+
def summary(self) -> Dict[str, Any]:
|
|
283
|
+
"""
|
|
284
|
+
Return a comprehensive summary dictionary.
|
|
285
|
+
|
|
286
|
+
Sections
|
|
287
|
+
--------
|
|
288
|
+
inventory, spatial, pixel_stats, quality, colour,
|
|
289
|
+
texture, frequency, duplicates, dataset_stats, labels
|
|
290
|
+
"""
|
|
291
|
+
self._check_loaded()
|
|
292
|
+
valid = [r for r in self._records if not r.is_corrupt]
|
|
293
|
+
corrupt = [r for r in self._records if r.is_corrupt]
|
|
294
|
+
|
|
295
|
+
if not valid:
|
|
296
|
+
return {"error": "No valid images found."}
|
|
297
|
+
|
|
298
|
+
def arr(attr):
|
|
299
|
+
return np.array([getattr(r, attr) for r in valid
|
|
300
|
+
if getattr(r, attr) is not None])
|
|
301
|
+
|
|
302
|
+
# ── inventory ───────────────────────────────────────────────
|
|
303
|
+
ext_dist = dict(Counter(r.file_ext for r in valid))
|
|
304
|
+
mode_dist = dict(Counter(
|
|
305
|
+
{1: "grayscale", 3: "RGB", 4: "RGBA"}.get(r.channels, str(r.channels))
|
|
306
|
+
for r in valid
|
|
307
|
+
))
|
|
308
|
+
dtype_dist = dict(Counter(r.dtype for r in valid))
|
|
309
|
+
|
|
310
|
+
# ── spatial ─────────────────────────────────────────────────
|
|
311
|
+
orientations = Counter()
|
|
312
|
+
for r in valid:
|
|
313
|
+
if r.aspect_ratio > 1.05: orientations["landscape"] += 1
|
|
314
|
+
elif r.aspect_ratio < 0.95: orientations["portrait"] += 1
|
|
315
|
+
else: orientations["square"] += 1
|
|
316
|
+
|
|
317
|
+
# ── pixel / colour ───────────────────────────────────────────
|
|
318
|
+
mean_rgb_matrix = np.array([r.mean_rgb for r in valid
|
|
319
|
+
if r.mean_rgb is not None])
|
|
320
|
+
dataset_mean = mean_rgb_matrix.mean(axis=0).tolist() \
|
|
321
|
+
if len(mean_rgb_matrix) else []
|
|
322
|
+
dataset_std_across = mean_rgb_matrix.std(axis=0).tolist() \
|
|
323
|
+
if len(mean_rgb_matrix) else []
|
|
324
|
+
|
|
325
|
+
std_rgb_matrix = np.array([r.std_rgb for r in valid
|
|
326
|
+
if r.std_rgb is not None])
|
|
327
|
+
dataset_pixel_std = std_rgb_matrix.mean(axis=0).tolist() \
|
|
328
|
+
if len(std_rgb_matrix) else []
|
|
329
|
+
|
|
330
|
+
color_temp_dist = dict(Counter(r.color_temp for r in valid
|
|
331
|
+
if r.color_temp))
|
|
332
|
+
grayscale_like_count = sum(r.is_grayscale_like for r in valid)
|
|
333
|
+
|
|
334
|
+
# ── quality ──────────────────────────────────────────────────
|
|
335
|
+
blurry_count = sum(r.is_blurry for r in valid)
|
|
336
|
+
overexp = arr("overexposed_frac")
|
|
337
|
+
underexp = arr("underexposed_frac")
|
|
338
|
+
|
|
339
|
+
# ── texture ──────────────────────────────────────────────────
|
|
340
|
+
texture_summary = {}
|
|
341
|
+
for feat in ("glcm_contrast", "glcm_dissimilarity",
|
|
342
|
+
"glcm_homogeneity", "glcm_energy",
|
|
343
|
+
"glcm_correlation", "glcm_asm"):
|
|
344
|
+
a = arr(feat)
|
|
345
|
+
if len(a):
|
|
346
|
+
texture_summary[feat] = _stat_dict(a)
|
|
347
|
+
|
|
348
|
+
# ── frequency ────────────────────────────────────────────────
|
|
349
|
+
freq_summary = {}
|
|
350
|
+
for band in ("freq_low", "freq_mid", "freq_high"):
|
|
351
|
+
a = arr(band)
|
|
352
|
+
if len(a):
|
|
353
|
+
freq_summary[band] = _stat_dict(a)
|
|
354
|
+
|
|
355
|
+
# ── duplicates ───────────────────────────────────────────────
|
|
356
|
+
exact_dupe_groups = self._find_exact_duplicates(valid)
|
|
357
|
+
near_dupe_groups = self._find_near_duplicates(valid)
|
|
358
|
+
|
|
359
|
+
# ── label info ───────────────────────────────────────────────
|
|
360
|
+
label_dist = None
|
|
361
|
+
class_imbalance_ratio = None
|
|
362
|
+
if any(r.label for r in valid):
|
|
363
|
+
lc = Counter(r.label for r in valid)
|
|
364
|
+
label_dist = dict(lc)
|
|
365
|
+
mx = max(lc.values()); mn = min(lc.values())
|
|
366
|
+
class_imbalance_ratio = round(mx / mn, 3) if mn else None
|
|
367
|
+
|
|
368
|
+
result = {
|
|
369
|
+
# ── inventory
|
|
370
|
+
"inventory": {
|
|
371
|
+
"total": len(self._records),
|
|
372
|
+
"valid": len(valid),
|
|
373
|
+
"corrupt": len(corrupt),
|
|
374
|
+
"corrupt_paths": [r.path for r in corrupt],
|
|
375
|
+
"format_distribution": ext_dist,
|
|
376
|
+
"colour_mode_distribution": mode_dist,
|
|
377
|
+
"dtype_distribution": dtype_dist,
|
|
378
|
+
},
|
|
379
|
+
# ── spatial
|
|
380
|
+
"spatial": {
|
|
381
|
+
"height": _stat_dict(arr("height")),
|
|
382
|
+
"width": _stat_dict(arr("width")),
|
|
383
|
+
"aspect_ratio": _stat_dict(arr("aspect_ratio")),
|
|
384
|
+
"megapixels": _stat_dict(arr("megapixels")),
|
|
385
|
+
"file_size_kb": _stat_dict(arr("file_size_kb")),
|
|
386
|
+
"orientation_distribution": dict(orientations),
|
|
387
|
+
},
|
|
388
|
+
# ── pixel stats
|
|
389
|
+
"pixel_stats": {
|
|
390
|
+
"dataset_mean_rgb": dataset_mean,
|
|
391
|
+
"dataset_std_rgb_across_images": dataset_std_across,
|
|
392
|
+
"dataset_pixel_std_rgb": dataset_pixel_std,
|
|
393
|
+
"per_image_mean_rgb": _stat_dict(mean_rgb_matrix.mean(axis=1))
|
|
394
|
+
if len(mean_rgb_matrix) else {},
|
|
395
|
+
},
|
|
396
|
+
# ── quality
|
|
397
|
+
"quality": {
|
|
398
|
+
"brightness": _stat_dict(arr("brightness")),
|
|
399
|
+
"contrast": _stat_dict(arr("contrast")),
|
|
400
|
+
"sharpness": _stat_dict(arr("sharpness")),
|
|
401
|
+
"noise_estimate": _stat_dict(arr("noise_estimate")),
|
|
402
|
+
"entropy": _stat_dict(arr("entropy_val")),
|
|
403
|
+
"compression_score": _stat_dict(arr("compression_score")),
|
|
404
|
+
"blurry_count": blurry_count,
|
|
405
|
+
"blurry_fraction": round(blurry_count / len(valid), 4),
|
|
406
|
+
"overexposed_frac": _stat_dict(overexp),
|
|
407
|
+
"underexposed_frac": _stat_dict(underexp),
|
|
408
|
+
},
|
|
409
|
+
# ── colour
|
|
410
|
+
"colour": {
|
|
411
|
+
"saturation": _stat_dict(arr("saturation_mean")),
|
|
412
|
+
"colour_temp_distribution": color_temp_dist,
|
|
413
|
+
"grayscale_like_count": grayscale_like_count,
|
|
414
|
+
"grayscale_like_fraction": round(grayscale_like_count / len(valid), 4),
|
|
415
|
+
},
|
|
416
|
+
# ── texture
|
|
417
|
+
"texture": texture_summary,
|
|
418
|
+
# ── frequency
|
|
419
|
+
"frequency": freq_summary,
|
|
420
|
+
# ── duplicates
|
|
421
|
+
"duplicates": {
|
|
422
|
+
"exact_duplicate_groups": exact_dupe_groups,
|
|
423
|
+
"n_exact_duplicate_groups": len(exact_dupe_groups),
|
|
424
|
+
"near_duplicate_groups": near_dupe_groups,
|
|
425
|
+
"n_near_duplicate_groups": len(near_dupe_groups),
|
|
426
|
+
},
|
|
427
|
+
# ── labels
|
|
428
|
+
"labels": {
|
|
429
|
+
"label_distribution": label_dist,
|
|
430
|
+
"class_imbalance_ratio": class_imbalance_ratio,
|
|
431
|
+
},
|
|
432
|
+
}
|
|
433
|
+
self._results["summary"] = result
|
|
434
|
+
return result
|
|
435
|
+
|
|
436
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
437
|
+
# Dataset-level normalisation stats
|
|
438
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
439
|
+
|
|
440
|
+
def normalization_stats(self) -> Dict[str, List[float]]:
|
|
441
|
+
"""
|
|
442
|
+
Compute per-channel mean and std for use in dataset normalisation
|
|
443
|
+
(e.g. ``torchvision.transforms.Normalize``).
|
|
444
|
+
|
|
445
|
+
Returns
|
|
446
|
+
-------
|
|
447
|
+
dict with keys ``mean`` and ``std``, each a list of 3 floats
|
|
448
|
+
in [0, 1] (channel order: R, G, B).
|
|
449
|
+
|
|
450
|
+
Example
|
|
451
|
+
-------
|
|
452
|
+
>>> stats = eda.normalization_stats()
|
|
453
|
+
>>> # use in torchvision:
|
|
454
|
+
>>> transforms.Normalize(mean=stats["mean"], std=stats["std"])
|
|
455
|
+
"""
|
|
456
|
+
self._check_loaded()
|
|
457
|
+
valid = [r for r in self._records if not r.is_corrupt
|
|
458
|
+
and r.mean_rgb is not None]
|
|
459
|
+
if not valid:
|
|
460
|
+
raise RuntimeError("No valid images to compute statistics from.")
|
|
461
|
+
|
|
462
|
+
means = np.array([r.mean_rgb for r in valid]) / 255.0 # (N, 3)
|
|
463
|
+
stds = np.array([r.std_rgb for r in valid]) / 255.0 # (N, 3)
|
|
464
|
+
|
|
465
|
+
return {
|
|
466
|
+
"mean": means.mean(axis=0).tolist(),
|
|
467
|
+
"std": stds.mean(axis=0).tolist(),
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
471
|
+
# Plotting — main dashboard
|
|
472
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
473
|
+
|
|
474
|
+
def plot(
|
|
475
|
+
self,
|
|
476
|
+
figsize: Tuple[int, int] = (24, 28),
|
|
477
|
+
save_path: Optional[str] = None,
|
|
478
|
+
dpi: int = 150,
|
|
479
|
+
) -> None:
|
|
480
|
+
"""Render the full EDA dashboard (6-row × 4-col grid)."""
|
|
481
|
+
self._check_loaded()
|
|
482
|
+
plt = _plt(); mpl = _mpl()
|
|
483
|
+
valid = [r for r in self._records if not r.is_corrupt]
|
|
484
|
+
|
|
485
|
+
fig = plt.figure(figsize=figsize, facecolor="white")
|
|
486
|
+
fig.suptitle("VisEDA — Image Dataset Analysis",
|
|
487
|
+
fontsize=24, color="#1f2328", y=0.99, fontweight="bold")
|
|
488
|
+
|
|
489
|
+
gs = mpl.gridspec.GridSpec(
|
|
490
|
+
6, 4, figure=fig,
|
|
491
|
+
hspace=0.55, wspace=0.35,
|
|
492
|
+
left=0.06, right=0.97, top=0.97, bottom=0.02,
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
def ax(*args, **kw):
|
|
496
|
+
a = fig.add_subplot(*args, **kw)
|
|
497
|
+
a.set_facecolor("#f6f8fa")
|
|
498
|
+
a.tick_params(colors="#57606a", labelsize=8)
|
|
499
|
+
for sp in a.spines.values():
|
|
500
|
+
sp.set_edgecolor("#d0d7de")
|
|
501
|
+
return a
|
|
502
|
+
|
|
503
|
+
# ── Row 0: inventory overview ─────────────────────────────────
|
|
504
|
+
self._plot_info_card(ax(gs[0, :2]), valid)
|
|
505
|
+
self._plot_label_dist(ax(gs[0, 2:]), valid)
|
|
506
|
+
|
|
507
|
+
# ── Row 1: spatial distributions ─────────────────────────────
|
|
508
|
+
self._plot_hist(ax(gs[1, 0]), [r.height for r in valid],
|
|
509
|
+
"Heights (px)", "#58a6ff")
|
|
510
|
+
self._plot_hist(ax(gs[1, 1]), [r.width for r in valid],
|
|
511
|
+
"Widths (px)", "#3fb950")
|
|
512
|
+
self._plot_hist(ax(gs[1, 2]), [r.aspect_ratio for r in valid],
|
|
513
|
+
"Aspect Ratios", "#d2a8ff")
|
|
514
|
+
self._plot_hist(ax(gs[1, 3]), [r.megapixels for r in valid],
|
|
515
|
+
"Megapixels", "#ffa657")
|
|
516
|
+
|
|
517
|
+
# ── Row 2: quality metrics ────────────────────────────────────
|
|
518
|
+
self._plot_hist(ax(gs[2, 0]), [r.brightness for r in valid],
|
|
519
|
+
"Brightness", "#79c0ff")
|
|
520
|
+
self._plot_hist(ax(gs[2, 1]), [r.contrast for r in valid],
|
|
521
|
+
"Contrast (std)", "#56d364")
|
|
522
|
+
self._plot_hist(ax(gs[2, 2]), [r.sharpness for r in valid],
|
|
523
|
+
"Sharpness (Laplacian var)", "#e3b341",
|
|
524
|
+
log_x=True)
|
|
525
|
+
self._plot_hist(ax(gs[2, 3]), [r.noise_estimate for r in valid],
|
|
526
|
+
"Noise Estimate", "#f78166")
|
|
527
|
+
|
|
528
|
+
# ── Row 3: exposure & colour quality ─────────────────────────
|
|
529
|
+
self._plot_hist(ax(gs[3, 0]),
|
|
530
|
+
[r.overexposed_frac * 100 for r in valid],
|
|
531
|
+
"Overexposed Pixels (%)", "#ff6b6b")
|
|
532
|
+
self._plot_hist(ax(gs[3, 1]),
|
|
533
|
+
[r.underexposed_frac * 100 for r in valid],
|
|
534
|
+
"Underexposed Pixels (%)", "#a5d8ff")
|
|
535
|
+
self._plot_hist(ax(gs[3, 2]), [r.saturation_mean for r in valid],
|
|
536
|
+
"Colour Saturation (HSV-S)", "#f9c74f")
|
|
537
|
+
self._plot_hist(ax(gs[3, 3]), [r.entropy_val for r in valid],
|
|
538
|
+
"Pixel Entropy (bits)", "#90be6d")
|
|
539
|
+
|
|
540
|
+
# ── Row 4: channel histograms + texture ──────────────────────
|
|
541
|
+
self._plot_channel_histograms(ax(gs[4, :2]), valid)
|
|
542
|
+
self._plot_texture_radar(ax(gs[4, 2:]), valid)
|
|
543
|
+
|
|
544
|
+
# ── Row 5: frequency + scatter + colour temp ──────────────────
|
|
545
|
+
self._plot_frequency_bands(ax(gs[5, 0]), valid)
|
|
546
|
+
self._plot_brightness_sharpness_scatter(ax(gs[5, 1]), valid)
|
|
547
|
+
self._plot_color_temp_pie(ax(gs[5, 2]), valid)
|
|
548
|
+
self._plot_format_dist(ax(gs[5, 3]), valid)
|
|
549
|
+
|
|
550
|
+
self._finalise(fig, save_path, dpi)
|
|
551
|
+
|
|
552
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
553
|
+
# Plotting — colour dashboard
|
|
554
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
555
|
+
|
|
556
|
+
def plot_colour(
|
|
557
|
+
self,
|
|
558
|
+
figsize: Tuple[int, int] = (22, 14),
|
|
559
|
+
save_path: Optional[str] = None,
|
|
560
|
+
dpi: int = 150,
|
|
561
|
+
) -> None:
|
|
562
|
+
"""Deep-dive colour analysis dashboard."""
|
|
563
|
+
self._check_loaded()
|
|
564
|
+
plt = _plt(); mpl = _mpl()
|
|
565
|
+
valid = [r for r in self._records if not r.is_corrupt]
|
|
566
|
+
|
|
567
|
+
fig = plt.figure(figsize=figsize, facecolor="white")
|
|
568
|
+
fig.suptitle("VisEDA — Colour Analysis",
|
|
569
|
+
fontsize=20, color="#1f2328", y=0.98, fontweight="bold")
|
|
570
|
+
|
|
571
|
+
gs = mpl.gridspec.GridSpec(2, 4, figure=fig,
|
|
572
|
+
hspace=0.45, wspace=0.35,
|
|
573
|
+
left=0.06, right=0.97,
|
|
574
|
+
top=0.93, bottom=0.06)
|
|
575
|
+
|
|
576
|
+
def ax(*args):
|
|
577
|
+
a = fig.add_subplot(*args)
|
|
578
|
+
a.set_facecolor("#f6f8fa")
|
|
579
|
+
a.tick_params(colors="#57606a", labelsize=8)
|
|
580
|
+
for sp in a.spines.values():
|
|
581
|
+
sp.set_edgecolor("#d0d7de")
|
|
582
|
+
return a
|
|
583
|
+
|
|
584
|
+
self._plot_channel_histograms(ax(gs[0, :2]), valid)
|
|
585
|
+
self._plot_dominant_colours(ax(gs[0, 2:]), valid)
|
|
586
|
+
self._plot_hsv_hue_wheel(ax(gs[1, 0]), valid)
|
|
587
|
+
self._plot_saturation_value(ax(gs[1, 1]), valid)
|
|
588
|
+
self._plot_lab_ab_scatter(ax(gs[1, 2]), valid)
|
|
589
|
+
self._plot_color_temp_pie(ax(gs[1, 3]), valid)
|
|
590
|
+
|
|
591
|
+
self._finalise(fig, save_path, dpi)
|
|
592
|
+
|
|
593
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
594
|
+
# Plotting — quality dashboard
|
|
595
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
596
|
+
|
|
597
|
+
def plot_quality(
|
|
598
|
+
self,
|
|
599
|
+
figsize: Tuple[int, int] = (22, 14),
|
|
600
|
+
save_path: Optional[str] = None,
|
|
601
|
+
dpi: int = 150,
|
|
602
|
+
) -> None:
|
|
603
|
+
"""Deep-dive quality metrics dashboard."""
|
|
604
|
+
self._check_loaded()
|
|
605
|
+
plt = _plt(); mpl = _mpl()
|
|
606
|
+
valid = [r for r in self._records if not r.is_corrupt]
|
|
607
|
+
|
|
608
|
+
fig = plt.figure(figsize=figsize, facecolor="white")
|
|
609
|
+
fig.suptitle("VisEDA — Quality Analysis",
|
|
610
|
+
fontsize=20, color="#1f2328", y=0.98, fontweight="bold")
|
|
611
|
+
|
|
612
|
+
gs = mpl.gridspec.GridSpec(2, 4, figure=fig,
|
|
613
|
+
hspace=0.45, wspace=0.35,
|
|
614
|
+
left=0.06, right=0.97,
|
|
615
|
+
top=0.93, bottom=0.06)
|
|
616
|
+
|
|
617
|
+
def ax(*args):
|
|
618
|
+
a = fig.add_subplot(*args)
|
|
619
|
+
a.set_facecolor("#f6f8fa")
|
|
620
|
+
a.tick_params(colors="#57606a", labelsize=8)
|
|
621
|
+
for sp in a.spines.values():
|
|
622
|
+
sp.set_edgecolor("#d0d7de")
|
|
623
|
+
return a
|
|
624
|
+
|
|
625
|
+
self._plot_hist(ax(gs[0, 0]), [r.sharpness for r in valid],
|
|
626
|
+
"Sharpness", "#e3b341", log_x=True)
|
|
627
|
+
self._plot_hist(ax(gs[0, 1]), [r.noise_estimate for r in valid],
|
|
628
|
+
"Noise Estimate", "#f78166")
|
|
629
|
+
self._plot_hist(ax(gs[0, 2]),
|
|
630
|
+
[r.compression_score for r in valid
|
|
631
|
+
if r.compression_score is not None],
|
|
632
|
+
"JPEG Blockiness Score", "#79c0ff")
|
|
633
|
+
self._plot_blurry_breakdown(ax(gs[0, 3]), valid)
|
|
634
|
+
self._plot_exposure_scatter(ax(gs[1, :2]), valid)
|
|
635
|
+
self._plot_texture_radar(ax(gs[1, 2:]), valid)
|
|
636
|
+
|
|
637
|
+
self._finalise(fig, save_path, dpi)
|
|
638
|
+
|
|
639
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
640
|
+
# Plotting — texture dashboard
|
|
641
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
642
|
+
|
|
643
|
+
def plot_texture(
|
|
644
|
+
self,
|
|
645
|
+
figsize: Tuple[int, int] = (22, 10),
|
|
646
|
+
save_path: Optional[str] = None,
|
|
647
|
+
dpi: int = 150,
|
|
648
|
+
) -> None:
|
|
649
|
+
"""GLCM texture features and frequency analysis dashboard."""
|
|
650
|
+
self._check_loaded()
|
|
651
|
+
plt = _plt(); mpl = _mpl()
|
|
652
|
+
valid = [r for r in self._records if not r.is_corrupt]
|
|
653
|
+
|
|
654
|
+
fig = plt.figure(figsize=figsize, facecolor="white")
|
|
655
|
+
fig.suptitle("VisEDA — Texture & Frequency Analysis",
|
|
656
|
+
fontsize=20, color="#1f2328", y=0.98, fontweight="bold")
|
|
657
|
+
|
|
658
|
+
gs = mpl.gridspec.GridSpec(1, 4, figure=fig,
|
|
659
|
+
hspace=0.4, wspace=0.35,
|
|
660
|
+
left=0.06, right=0.97,
|
|
661
|
+
top=0.88, bottom=0.1)
|
|
662
|
+
|
|
663
|
+
def ax(*args):
|
|
664
|
+
a = fig.add_subplot(*args)
|
|
665
|
+
a.set_facecolor("#f6f8fa")
|
|
666
|
+
a.tick_params(colors="#57606a", labelsize=8)
|
|
667
|
+
for sp in a.spines.values():
|
|
668
|
+
sp.set_edgecolor("#d0d7de")
|
|
669
|
+
return a
|
|
670
|
+
|
|
671
|
+
for i, (feat, color) in enumerate([
|
|
672
|
+
("glcm_contrast", "#58a6ff"),
|
|
673
|
+
("glcm_homogeneity", "#3fb950"),
|
|
674
|
+
("glcm_energy", "#e3b341"),
|
|
675
|
+
("glcm_correlation", "#d2a8ff"),
|
|
676
|
+
]):
|
|
677
|
+
self._plot_hist(ax(gs[0, i]),
|
|
678
|
+
[getattr(r, feat) for r in valid
|
|
679
|
+
if getattr(r, feat) is not None],
|
|
680
|
+
feat.replace("_", " ").title(), color)
|
|
681
|
+
|
|
682
|
+
self._finalise(fig, save_path, dpi)
|
|
683
|
+
|
|
684
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
685
|
+
# Plotting — sample grid
|
|
686
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
687
|
+
|
|
688
|
+
def plot_samples(
|
|
689
|
+
self,
|
|
690
|
+
n: int = 25,
|
|
691
|
+
cols: int = 5,
|
|
692
|
+
label: Optional[str] = None,
|
|
693
|
+
random_seed: int = 42,
|
|
694
|
+
figsize: Optional[Tuple] = None,
|
|
695
|
+
save_path: Optional[str] = None,
|
|
696
|
+
) -> None:
|
|
697
|
+
"""
|
|
698
|
+
Display a grid of sample images.
|
|
699
|
+
|
|
700
|
+
Parameters
|
|
701
|
+
----------
|
|
702
|
+
label
|
|
703
|
+
If supplied, only show images with this label.
|
|
704
|
+
"""
|
|
705
|
+
self._check_loaded()
|
|
706
|
+
plt = _plt(); cv2 = _cv2()
|
|
707
|
+
|
|
708
|
+
pool = [r for r in self._records
|
|
709
|
+
if not r.is_corrupt and not r.path.startswith("<array")]
|
|
710
|
+
if label:
|
|
711
|
+
pool = [r for r in pool if r.label == label]
|
|
712
|
+
|
|
713
|
+
rng = np.random.default_rng(random_seed)
|
|
714
|
+
sample = rng.choice(pool, size=min(n, len(pool)), replace=False)
|
|
715
|
+
rows = int(np.ceil(len(sample) / cols))
|
|
716
|
+
figsize = figsize or (cols * 3, rows * 3 + 0.5)
|
|
717
|
+
|
|
718
|
+
fig, axes = plt.subplots(rows, cols, figsize=figsize,
|
|
719
|
+
facecolor="white")
|
|
720
|
+
axes = np.array(axes).flatten()
|
|
721
|
+
|
|
722
|
+
for i, rec in enumerate(sample):
|
|
723
|
+
img = cv2.imread(str(rec.path))
|
|
724
|
+
if img is None:
|
|
725
|
+
axes[i].axis("off"); continue
|
|
726
|
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
|
727
|
+
img = _resize_display(img, 224)
|
|
728
|
+
axes[i].imshow(img)
|
|
729
|
+
title = (rec.label or Path(rec.path).stem)[:20]
|
|
730
|
+
axes[i].set_title(title, fontsize=7, color="#1f2328")
|
|
731
|
+
axes[i].axis("off")
|
|
732
|
+
|
|
733
|
+
for j in range(len(sample), len(axes)):
|
|
734
|
+
axes[j].axis("off")
|
|
735
|
+
|
|
736
|
+
title_str = f"Sample Images" + (f" — {label}" if label else "")
|
|
737
|
+
fig.suptitle(title_str, color="#1f2328", fontsize=13)
|
|
738
|
+
plt.tight_layout()
|
|
739
|
+
self._finalise(fig, save_path, dpi=120)
|
|
740
|
+
|
|
741
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
742
|
+
# Plotting — per-class sample grids
|
|
743
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
744
|
+
|
|
745
|
+
def plot_class_samples(
|
|
746
|
+
self,
|
|
747
|
+
n_per_class: int = 5,
|
|
748
|
+
save_path: Optional[str] = None,
|
|
749
|
+
) -> None:
|
|
750
|
+
"""One row of sample images per class label."""
|
|
751
|
+
self._check_loaded()
|
|
752
|
+
plt = _plt(); cv2 = _cv2()
|
|
753
|
+
|
|
754
|
+
classes = sorted(set(r.label for r in self._records
|
|
755
|
+
if r.label and not r.is_corrupt))
|
|
756
|
+
if not classes:
|
|
757
|
+
self._log("No labels found — use label_from_parent=True when loading.")
|
|
758
|
+
return
|
|
759
|
+
|
|
760
|
+
n_classes = len(classes)
|
|
761
|
+
fig, axes = plt.subplots(
|
|
762
|
+
n_classes, n_per_class,
|
|
763
|
+
figsize=(n_per_class * 2.5, n_classes * 2.5),
|
|
764
|
+
facecolor="white",
|
|
765
|
+
)
|
|
766
|
+
if n_classes == 1:
|
|
767
|
+
axes = axes[np.newaxis, :]
|
|
768
|
+
|
|
769
|
+
rng = np.random.default_rng(0)
|
|
770
|
+
for row, cls in enumerate(classes):
|
|
771
|
+
pool = [r for r in self._records
|
|
772
|
+
if r.label == cls and not r.is_corrupt
|
|
773
|
+
and not r.path.startswith("<array")]
|
|
774
|
+
sample = rng.choice(pool, size=min(n_per_class, len(pool)),
|
|
775
|
+
replace=False)
|
|
776
|
+
for col in range(n_per_class):
|
|
777
|
+
ax = axes[row, col]
|
|
778
|
+
ax.axis("off")
|
|
779
|
+
ax.set_facecolor("white")
|
|
780
|
+
if col >= len(sample):
|
|
781
|
+
continue
|
|
782
|
+
img = cv2.imread(str(sample[col].path))
|
|
783
|
+
if img is None:
|
|
784
|
+
continue
|
|
785
|
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
|
786
|
+
img = _resize_display(img, 128)
|
|
787
|
+
ax.imshow(img)
|
|
788
|
+
if col == 0:
|
|
789
|
+
ax.set_ylabel(cls, color="#1f2328", fontsize=8,
|
|
790
|
+
rotation=0, labelpad=50, va="center")
|
|
791
|
+
|
|
792
|
+
fig.suptitle("Per-class Sample Images", color="#1f2328", fontsize=14)
|
|
793
|
+
plt.tight_layout()
|
|
794
|
+
self._finalise(fig, save_path, dpi=120)
|
|
795
|
+
|
|
796
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
797
|
+
# Plotting — average image
|
|
798
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
799
|
+
|
|
800
|
+
def plot_average_image(
|
|
801
|
+
self,
|
|
802
|
+
target_size: Tuple[int, int] = (224, 224),
|
|
803
|
+
label: Optional[str] = None,
|
|
804
|
+
save_path: Optional[str] = None,
|
|
805
|
+
) -> None:
|
|
806
|
+
"""
|
|
807
|
+
Compute and display the pixel-wise average image across the dataset.
|
|
808
|
+
|
|
809
|
+
This reveals systematic dataset biases (e.g. sky always at top,
|
|
810
|
+
ground at bottom, objects centred).
|
|
811
|
+
"""
|
|
812
|
+
self._check_loaded()
|
|
813
|
+
plt = _plt(); cv2 = _cv2()
|
|
814
|
+
|
|
815
|
+
pool = [r for r in self._records
|
|
816
|
+
if not r.is_corrupt and not r.path.startswith("<array")]
|
|
817
|
+
if label:
|
|
818
|
+
pool = [r for r in pool if r.label == label]
|
|
819
|
+
|
|
820
|
+
if not pool:
|
|
821
|
+
self._log("No images available for average-image computation.")
|
|
822
|
+
return
|
|
823
|
+
|
|
824
|
+
self._log(f"Computing average image over {len(pool)} images …")
|
|
825
|
+
acc = np.zeros((*target_size, 3), dtype=np.float64)
|
|
826
|
+
count = 0
|
|
827
|
+
for rec in pool:
|
|
828
|
+
img = cv2.imread(str(rec.path))
|
|
829
|
+
if img is None:
|
|
830
|
+
continue
|
|
831
|
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
|
832
|
+
img = cv2.resize(img, (target_size[1], target_size[0]))
|
|
833
|
+
acc += img.astype(np.float64)
|
|
834
|
+
count += 1
|
|
835
|
+
|
|
836
|
+
if count == 0:
|
|
837
|
+
self._log("No images could be read.")
|
|
838
|
+
return
|
|
839
|
+
|
|
840
|
+
avg = (acc / count).astype(np.uint8)
|
|
841
|
+
|
|
842
|
+
fig, axes = plt.subplots(1, 2, figsize=(10, 5), facecolor="white")
|
|
843
|
+
axes[0].imshow(avg)
|
|
844
|
+
axes[0].set_title("Average Image", color="#1f2328", fontsize=12)
|
|
845
|
+
axes[0].axis("off")
|
|
846
|
+
|
|
847
|
+
# standard deviation image
|
|
848
|
+
self._log("Computing std-dev image …")
|
|
849
|
+
sq_acc = np.zeros_like(acc)
|
|
850
|
+
for rec in pool:
|
|
851
|
+
img = cv2.imread(str(rec.path))
|
|
852
|
+
if img is None:
|
|
853
|
+
continue
|
|
854
|
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
|
855
|
+
img = cv2.resize(img, (target_size[1], target_size[0]))
|
|
856
|
+
sq_acc += (img.astype(np.float64) - acc / count) ** 2
|
|
857
|
+
|
|
858
|
+
std_img = np.sqrt(sq_acc / count).astype(np.uint8)
|
|
859
|
+
axes[1].imshow(std_img)
|
|
860
|
+
axes[1].set_title("Std-Dev Image\n(high = high variance across dataset)",
|
|
861
|
+
color="#1f2328", fontsize=12)
|
|
862
|
+
axes[1].axis("off")
|
|
863
|
+
|
|
864
|
+
title = f"Dataset Average & Variance" + (f" — {label}" if label else "")
|
|
865
|
+
fig.suptitle(title, color="#1f2328", fontsize=14)
|
|
866
|
+
plt.tight_layout()
|
|
867
|
+
self._finalise(fig, save_path, dpi=150)
|
|
868
|
+
|
|
869
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
870
|
+
# Plotting — channel correlation
|
|
871
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
872
|
+
|
|
873
|
+
def plot_channel_correlation(
|
|
874
|
+
self,
|
|
875
|
+
max_pixels: int = 50_000,
|
|
876
|
+
save_path: Optional[str] = None,
|
|
877
|
+
) -> None:
|
|
878
|
+
"""R/G/B pairwise scatter matrix across a random pixel sample."""
|
|
879
|
+
self._check_loaded()
|
|
880
|
+
plt = _plt(); cv2 = _cv2()
|
|
881
|
+
|
|
882
|
+
pool = [r for r in self._records if not r.is_corrupt
|
|
883
|
+
and not r.path.startswith("<array")][:80]
|
|
884
|
+
|
|
885
|
+
rng = np.random.default_rng(0)
|
|
886
|
+
pixels = []
|
|
887
|
+
for rec in pool:
|
|
888
|
+
img = cv2.imread(str(rec.path))
|
|
889
|
+
if img is None:
|
|
890
|
+
continue
|
|
891
|
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
|
892
|
+
flat = img.reshape(-1, 3)
|
|
893
|
+
n = max_pixels // len(pool)
|
|
894
|
+
if len(flat) > n:
|
|
895
|
+
flat = flat[rng.choice(len(flat), n, replace=False)]
|
|
896
|
+
pixels.append(flat)
|
|
897
|
+
|
|
898
|
+
if not pixels:
|
|
899
|
+
return
|
|
900
|
+
pixels = np.vstack(pixels).astype(float)
|
|
901
|
+
ch_names = ["Red", "Green", "Blue"]
|
|
902
|
+
ch_colors = ["#ff6b6b", "#51cf66", "#339af0"]
|
|
903
|
+
|
|
904
|
+
fig, axes = plt.subplots(3, 3, figsize=(11, 11), facecolor="white")
|
|
905
|
+
for i in range(3):
|
|
906
|
+
for j in range(3):
|
|
907
|
+
a = axes[i, j]
|
|
908
|
+
a.set_facecolor("#f6f8fa")
|
|
909
|
+
for sp in a.spines.values():
|
|
910
|
+
sp.set_edgecolor("#d0d7de")
|
|
911
|
+
a.tick_params(colors="#57606a", labelsize=7)
|
|
912
|
+
if i == j:
|
|
913
|
+
a.hist(pixels[:, i], bins=60, color=ch_colors[i], alpha=0.85)
|
|
914
|
+
a.set_title(ch_names[i], color="#1f2328", fontsize=9)
|
|
915
|
+
else:
|
|
916
|
+
a.scatter(pixels[:, j], pixels[:, i],
|
|
917
|
+
alpha=0.04, s=1, color=ch_colors[i])
|
|
918
|
+
corr = float(np.corrcoef(pixels[:, i], pixels[:, j])[0, 1])
|
|
919
|
+
a.set_title(f"r = {corr:.3f}", color="#57606a", fontsize=8)
|
|
920
|
+
if i == 2:
|
|
921
|
+
a.set_xlabel(ch_names[j], color="#57606a", fontsize=8)
|
|
922
|
+
if j == 0:
|
|
923
|
+
a.set_ylabel(ch_names[i], color="#57606a", fontsize=8)
|
|
924
|
+
|
|
925
|
+
fig.suptitle("Channel Correlation Matrix", color="#1f2328", fontsize=14)
|
|
926
|
+
plt.tight_layout()
|
|
927
|
+
self._finalise(fig, save_path, dpi=130)
|
|
928
|
+
|
|
929
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
930
|
+
# Plotting — duplicate viewer
|
|
931
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
932
|
+
|
|
933
|
+
def plot_duplicates(
|
|
934
|
+
self,
|
|
935
|
+
mode: str = "near",
|
|
936
|
+
max_groups: int = 5,
|
|
937
|
+
save_path: Optional[str] = None,
|
|
938
|
+
) -> None:
|
|
939
|
+
"""
|
|
940
|
+
Visualise duplicate / near-duplicate groups.
|
|
941
|
+
|
|
942
|
+
Parameters
|
|
943
|
+
----------
|
|
944
|
+
mode : ``"exact"`` | ``"near"``
|
|
945
|
+
"""
|
|
946
|
+
self._check_loaded()
|
|
947
|
+
plt = _plt(); cv2 = _cv2()
|
|
948
|
+
|
|
949
|
+
valid = [r for r in self._records if not r.is_corrupt]
|
|
950
|
+
groups = (self._find_exact_duplicates(valid) if mode == "exact"
|
|
951
|
+
else self._find_near_duplicates(valid))
|
|
952
|
+
groups = groups[:max_groups]
|
|
953
|
+
|
|
954
|
+
if not groups:
|
|
955
|
+
self._log(f"No {mode} duplicates found.")
|
|
956
|
+
return
|
|
957
|
+
|
|
958
|
+
max_cols = max(len(g) for g in groups)
|
|
959
|
+
fig, axes = plt.subplots(
|
|
960
|
+
len(groups), max_cols,
|
|
961
|
+
figsize=(max_cols * 2.5, len(groups) * 2.5),
|
|
962
|
+
facecolor="white",
|
|
963
|
+
squeeze=False,
|
|
964
|
+
)
|
|
965
|
+
|
|
966
|
+
for row, group in enumerate(groups):
|
|
967
|
+
for col in range(max_cols):
|
|
968
|
+
ax = axes[row, col]
|
|
969
|
+
ax.axis("off")
|
|
970
|
+
ax.set_facecolor("white")
|
|
971
|
+
if col >= len(group):
|
|
972
|
+
continue
|
|
973
|
+
img = cv2.imread(str(group[col]))
|
|
974
|
+
if img is None:
|
|
975
|
+
continue
|
|
976
|
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
|
977
|
+
img = _resize_display(img, 128)
|
|
978
|
+
ax.imshow(img)
|
|
979
|
+
ax.set_title(Path(group[col]).name[:18],
|
|
980
|
+
fontsize=6, color="#57606a")
|
|
981
|
+
|
|
982
|
+
fig.suptitle(f"{mode.capitalize()} Duplicate Groups",
|
|
983
|
+
color="#1f2328", fontsize=13)
|
|
984
|
+
plt.tight_layout()
|
|
985
|
+
self._finalise(fig, save_path, dpi=120)
|
|
986
|
+
|
|
987
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
988
|
+
# Report
|
|
989
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
990
|
+
|
|
991
|
+
def report(self, output_path: str = "viseda_report.html") -> str:
|
|
992
|
+
"""Generate a self-contained HTML report."""
|
|
993
|
+
self._check_loaded()
|
|
994
|
+
s = self.summary()
|
|
995
|
+
norm = self.normalization_stats()
|
|
996
|
+
s["normalization_stats"] = norm
|
|
997
|
+
_generate_html_report(s, output_path)
|
|
998
|
+
self._log(f"Report saved → {output_path}")
|
|
999
|
+
return output_path
|
|
1000
|
+
|
|
1001
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
1002
|
+
# Per-image analysis
|
|
1003
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
1004
|
+
|
|
1005
|
+
def _analyse_single(self, path: Path, label_from_parent: bool) -> ImageRecord:
|
|
1006
|
+
cv2 = _cv2()
|
|
1007
|
+
rec = ImageRecord()
|
|
1008
|
+
rec.path = str(path)
|
|
1009
|
+
rec.file_ext = path.suffix.lower()
|
|
1010
|
+
rec.file_size_kb = path.stat().st_size / 1024
|
|
1011
|
+
|
|
1012
|
+
if label_from_parent:
|
|
1013
|
+
rec.label = path.parent.name
|
|
1014
|
+
else:
|
|
1015
|
+
rec.label = self._labels_map.get(str(path.resolve()))
|
|
1016
|
+
|
|
1017
|
+
img_bgr = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
|
|
1018
|
+
if img_bgr is None:
|
|
1019
|
+
rec.is_corrupt = True
|
|
1020
|
+
return rec
|
|
1021
|
+
|
|
1022
|
+
# Normalise to HxWx3 uint8
|
|
1023
|
+
if img_bgr.ndim == 2:
|
|
1024
|
+
img_bgr = cv2.cvtColor(img_bgr, cv2.COLOR_GRAY2BGR)
|
|
1025
|
+
elif img_bgr.shape[2] == 4:
|
|
1026
|
+
img_bgr = cv2.cvtColor(img_bgr, cv2.COLOR_BGRA2BGR)
|
|
1027
|
+
|
|
1028
|
+
img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
|
|
1029
|
+
self._fill_stats(rec, img_rgb)
|
|
1030
|
+
return rec
|
|
1031
|
+
|
|
1032
|
+
def _fill_stats(self, rec: ImageRecord, img: np.ndarray) -> None:
|
|
1033
|
+
cv2 = _cv2()
|
|
1034
|
+
|
|
1035
|
+
if img.ndim == 2:
|
|
1036
|
+
img = np.stack([img, img, img], axis=-1)
|
|
1037
|
+
img = img.astype(np.uint8)
|
|
1038
|
+
|
|
1039
|
+
h, w = img.shape[:2]
|
|
1040
|
+
rec.height = h
|
|
1041
|
+
rec.width = w
|
|
1042
|
+
rec.channels = img.shape[2] if img.ndim == 3 else 1
|
|
1043
|
+
rec.dtype = str(img.dtype)
|
|
1044
|
+
rec.aspect_ratio = round(w / h, 4)
|
|
1045
|
+
rec.megapixels = round(h * w / 1_000_000, 4)
|
|
1046
|
+
|
|
1047
|
+
f = img.astype(np.float32)
|
|
1048
|
+
|
|
1049
|
+
# ── pixel stats ─────────────────────────────────────────────
|
|
1050
|
+
rec.mean_rgb = f.mean(axis=(0, 1)).tolist()
|
|
1051
|
+
rec.std_rgb = f.std(axis=(0, 1)).tolist()
|
|
1052
|
+
rec.min_rgb = f.min(axis=(0, 1)).tolist()
|
|
1053
|
+
rec.max_rgb = f.max(axis=(0, 1)).tolist()
|
|
1054
|
+
|
|
1055
|
+
# ── brightness & contrast (luminance) ────────────────────────
|
|
1056
|
+
gray = (0.299 * f[:, :, 0] + 0.587 * f[:, :, 1]
|
|
1057
|
+
+ 0.114 * f[:, :, 2])
|
|
1058
|
+
rec.brightness = float(gray.mean())
|
|
1059
|
+
rec.contrast = float(gray.std())
|
|
1060
|
+
|
|
1061
|
+
# ── sharpness: Laplacian variance ────────────────────────────
|
|
1062
|
+
gray_u8 = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
|
|
1063
|
+
lap = cv2.Laplacian(gray_u8, cv2.CV_64F)
|
|
1064
|
+
rec.sharpness = float(lap.var())
|
|
1065
|
+
rec.is_blurry = rec.sharpness < self.blur_threshold
|
|
1066
|
+
|
|
1067
|
+
# ── noise estimate: high-freq residual ───────────────────────
|
|
1068
|
+
blurred = cv2.GaussianBlur(gray_u8, (5, 5), 0).astype(np.float32)
|
|
1069
|
+
residual = gray_u8.astype(np.float32) - blurred
|
|
1070
|
+
rec.noise_estimate = float(residual.std())
|
|
1071
|
+
|
|
1072
|
+
# ── exposure ─────────────────────────────────────────────────
|
|
1073
|
+
rec.overexposed_frac = float((gray > 245).mean())
|
|
1074
|
+
rec.underexposed_frac = float((gray < 10).mean())
|
|
1075
|
+
|
|
1076
|
+
# ── entropy ──────────────────────────────────────────────────
|
|
1077
|
+
hist, _ = np.histogram(gray_u8.ravel(), bins=256, range=(0, 256))
|
|
1078
|
+
p = hist / (hist.sum() + 1e-12)
|
|
1079
|
+
p = p[p > 0]
|
|
1080
|
+
rec.entropy_val = float(-np.sum(p * np.log2(p)))
|
|
1081
|
+
|
|
1082
|
+
# ── JPEG blockiness score ────────────────────────────────────
|
|
1083
|
+
rec.compression_score = self._blockiness(gray_u8)
|
|
1084
|
+
|
|
1085
|
+
# ── colour spaces ────────────────────────────────────────────
|
|
1086
|
+
hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV).astype(np.float32)
|
|
1087
|
+
rec.mean_hsv = hsv.mean(axis=(0, 1)).tolist()
|
|
1088
|
+
rec.saturation_mean = float(hsv[:, :, 1].mean())
|
|
1089
|
+
rec.is_grayscale_like = rec.saturation_mean < 15.0
|
|
1090
|
+
|
|
1091
|
+
lab = cv2.cvtColor(img, cv2.COLOR_RGB2Lab).astype(np.float32)
|
|
1092
|
+
rec.mean_lab = lab.mean(axis=(0, 1)).tolist()
|
|
1093
|
+
|
|
1094
|
+
# colour temperature from R/B ratio
|
|
1095
|
+
r_mean = rec.mean_rgb[0]; b_mean = rec.mean_rgb[2]
|
|
1096
|
+
ratio = r_mean / (b_mean + 1e-6)
|
|
1097
|
+
if ratio > 1.1: rec.color_temp = "warm"
|
|
1098
|
+
elif ratio < 0.9: rec.color_temp = "cool"
|
|
1099
|
+
else: rec.color_temp = "neutral"
|
|
1100
|
+
|
|
1101
|
+
# ── perceptual hash (pHash) ──────────────────────────────────
|
|
1102
|
+
small = cv2.resize(gray_u8, (32, 32),
|
|
1103
|
+
interpolation=cv2.INTER_AREA).astype(np.float32)
|
|
1104
|
+
dct = cv2.dct(small)
|
|
1105
|
+
dct_low = dct[:8, :8].flatten()
|
|
1106
|
+
med = np.median(dct_low)
|
|
1107
|
+
rec.phash = int("".join("1" if v > med else "0"
|
|
1108
|
+
for v in dct_low), 2)
|
|
1109
|
+
|
|
1110
|
+
# average hash (aHash)
|
|
1111
|
+
tiny = cv2.resize(gray_u8, (8, 8),
|
|
1112
|
+
interpolation=cv2.INTER_AREA).astype(np.float32)
|
|
1113
|
+
mean_tiny = tiny.mean()
|
|
1114
|
+
rec.ahash = int("".join("1" if v > mean_tiny else "0"
|
|
1115
|
+
for v in tiny.flatten()), 2)
|
|
1116
|
+
|
|
1117
|
+
# MD5 (for exact duplicates)
|
|
1118
|
+
raw = img.tobytes()
|
|
1119
|
+
rec.md5 = hashlib.md5(raw).hexdigest()
|
|
1120
|
+
|
|
1121
|
+
# ── GLCM texture ─────────────────────────────────────────────
|
|
1122
|
+
if self.compute_glcm:
|
|
1123
|
+
self._fill_glcm(rec, gray_u8)
|
|
1124
|
+
|
|
1125
|
+
# ── FFT frequency bands ───────────────────────────────────────
|
|
1126
|
+
if self.compute_freq:
|
|
1127
|
+
self._fill_frequency(rec, gray_u8)
|
|
1128
|
+
|
|
1129
|
+
def _fill_glcm(self, rec: ImageRecord, gray_u8: np.ndarray) -> None:
|
|
1130
|
+
try:
|
|
1131
|
+
graycomatrix, graycoprops = _ski_feature()
|
|
1132
|
+
# Downsample for speed
|
|
1133
|
+
small = _cv2().resize(gray_u8, (128, 128))
|
|
1134
|
+
# Quantise to 64 levels
|
|
1135
|
+
quant = (small // 4).astype(np.uint8)
|
|
1136
|
+
glcm = graycomatrix(quant, distances=[1],
|
|
1137
|
+
angles=[0, np.pi / 4, np.pi / 2, 3 * np.pi / 4],
|
|
1138
|
+
levels=64, symmetric=True, normed=True)
|
|
1139
|
+
rec.glcm_contrast = float(graycoprops(glcm, "contrast").mean())
|
|
1140
|
+
rec.glcm_dissimilarity = float(graycoprops(glcm, "dissimilarity").mean())
|
|
1141
|
+
rec.glcm_homogeneity = float(graycoprops(glcm, "homogeneity").mean())
|
|
1142
|
+
rec.glcm_energy = float(graycoprops(glcm, "energy").mean())
|
|
1143
|
+
rec.glcm_correlation = float(graycoprops(glcm, "correlation").mean())
|
|
1144
|
+
rec.glcm_asm = float(graycoprops(glcm, "ASM").mean())
|
|
1145
|
+
except Exception:
|
|
1146
|
+
pass
|
|
1147
|
+
|
|
1148
|
+
def _fill_frequency(self, rec: ImageRecord, gray_u8: np.ndarray) -> None:
|
|
1149
|
+
try:
|
|
1150
|
+
small = _cv2().resize(gray_u8, (128, 128)).astype(np.float32)
|
|
1151
|
+
fft = np.fft.fft2(small)
|
|
1152
|
+
fft_s = np.fft.fftshift(np.abs(fft))
|
|
1153
|
+
h, w = fft_s.shape
|
|
1154
|
+
cy, cx = h // 2, w // 2
|
|
1155
|
+
total = fft_s.sum() + 1e-9
|
|
1156
|
+
|
|
1157
|
+
# Low = inner 10 %, Mid = 10–40 %, High = outer 40–50 %
|
|
1158
|
+
Y, X = np.ogrid[:h, :w]
|
|
1159
|
+
dist = np.sqrt((Y - cy) ** 2 + (X - cx) ** 2)
|
|
1160
|
+
max_d = min(cy, cx)
|
|
1161
|
+
|
|
1162
|
+
rec.freq_low = float(fft_s[dist < max_d * 0.10].sum() / total)
|
|
1163
|
+
rec.freq_mid = float(fft_s[(dist >= max_d * 0.10)
|
|
1164
|
+
& (dist < max_d * 0.40)].sum() / total)
|
|
1165
|
+
rec.freq_high = float(fft_s[dist >= max_d * 0.40].sum() / total)
|
|
1166
|
+
except Exception:
|
|
1167
|
+
pass
|
|
1168
|
+
|
|
1169
|
+
@staticmethod
|
|
1170
|
+
def _blockiness(gray_u8: np.ndarray) -> float:
|
|
1171
|
+
"""Estimate JPEG blockiness as mean absolute difference across 8-px boundaries."""
|
|
1172
|
+
h, w = gray_u8.shape
|
|
1173
|
+
g = gray_u8.astype(np.float32)
|
|
1174
|
+
scores = []
|
|
1175
|
+
for y in range(8, h, 8):
|
|
1176
|
+
scores.append(np.abs(g[y, :] - g[y - 1, :]).mean())
|
|
1177
|
+
for x in range(8, w, 8):
|
|
1178
|
+
scores.append(np.abs(g[:, x] - g[:, x - 1]).mean())
|
|
1179
|
+
return float(np.mean(scores)) if scores else 0.0
|
|
1180
|
+
|
|
1181
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
1182
|
+
# Duplicate detection
|
|
1183
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
1184
|
+
|
|
1185
|
+
def _find_exact_duplicates(
|
|
1186
|
+
self, records: List[ImageRecord]
|
|
1187
|
+
) -> List[List[str]]:
|
|
1188
|
+
groups: Dict[str, List[str]] = defaultdict(list)
|
|
1189
|
+
for r in records:
|
|
1190
|
+
if r.md5:
|
|
1191
|
+
groups[r.md5].append(r.path)
|
|
1192
|
+
return [g for g in groups.values() if len(g) > 1]
|
|
1193
|
+
|
|
1194
|
+
def _find_near_duplicates(
|
|
1195
|
+
self, records: List[ImageRecord]
|
|
1196
|
+
) -> List[List[str]]:
|
|
1197
|
+
groups: List[List[str]] = []
|
|
1198
|
+
visited: set = set()
|
|
1199
|
+
hashes = [(r.phash, r.path) for r in records if r.phash is not None]
|
|
1200
|
+
for i, (h1, p1) in enumerate(hashes):
|
|
1201
|
+
if p1 in visited:
|
|
1202
|
+
continue
|
|
1203
|
+
grp = [p1]
|
|
1204
|
+
for h2, p2 in hashes[i + 1:]:
|
|
1205
|
+
if p2 in visited:
|
|
1206
|
+
continue
|
|
1207
|
+
if bin(h1 ^ h2).count("1") <= self.phash_threshold:
|
|
1208
|
+
grp.append(p2)
|
|
1209
|
+
visited.add(p2)
|
|
1210
|
+
if len(grp) > 1:
|
|
1211
|
+
groups.append(grp)
|
|
1212
|
+
visited.add(p1)
|
|
1213
|
+
return groups
|
|
1214
|
+
|
|
1215
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
1216
|
+
# Plot helpers
|
|
1217
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
1218
|
+
|
|
1219
|
+
def _plot_info_card(self, ax, valid):
|
|
1220
|
+
ax.axis("off")
|
|
1221
|
+
corrupt = len(self._records) - len(valid)
|
|
1222
|
+
blurry = sum(r.is_blurry for r in valid)
|
|
1223
|
+
grey_like = sum(r.is_grayscale_like for r in valid)
|
|
1224
|
+
n_exact = len(self._find_exact_duplicates(valid))
|
|
1225
|
+
n_near = len(self._find_near_duplicates(valid))
|
|
1226
|
+
lines = [
|
|
1227
|
+
f"Total images: {len(self._records):,}",
|
|
1228
|
+
f"Valid / Corrupt: {len(valid):,} / {corrupt:,}",
|
|
1229
|
+
f"Unique labels: "
|
|
1230
|
+
f"{len(set(r.label for r in valid if r.label)):,}",
|
|
1231
|
+
f"Blurry images: {blurry:,} "
|
|
1232
|
+
f"({blurry/max(len(valid),1)*100:.1f}%)",
|
|
1233
|
+
f"Greyscale-like: {grey_like:,} "
|
|
1234
|
+
f"({grey_like/max(len(valid),1)*100:.1f}%)",
|
|
1235
|
+
f"Exact dupes groups: {n_exact:,}",
|
|
1236
|
+
f"Near dupes groups: {n_near:,}",
|
|
1237
|
+
f"Median H × W: "
|
|
1238
|
+
f"{int(np.median([r.height for r in valid]))} × "
|
|
1239
|
+
f"{int(np.median([r.width for r in valid]))}",
|
|
1240
|
+
]
|
|
1241
|
+
ax.text(0.04, 0.96, "\n".join(lines),
|
|
1242
|
+
transform=ax.transAxes, va="top", ha="left",
|
|
1243
|
+
fontsize=9.5, color="#1f2328", fontfamily="monospace",
|
|
1244
|
+
bbox=dict(boxstyle="round,pad=0.5", facecolor="#eaeef2",
|
|
1245
|
+
edgecolor="#d0d7de"))
|
|
1246
|
+
ax.set_title("Dataset Overview", color="#1f2328", fontsize=11)
|
|
1247
|
+
|
|
1248
|
+
def _plot_label_dist(self, ax, valid):
|
|
1249
|
+
labels = [r.label for r in valid if r.label]
|
|
1250
|
+
if not labels:
|
|
1251
|
+
ax.text(0.5, 0.5, "No labels provided\n(use label_from_parent=True)",
|
|
1252
|
+
ha="center", va="center", color="#57606a",
|
|
1253
|
+
transform=ax.transAxes, fontsize=9)
|
|
1254
|
+
ax.set_title("Label Distribution", color="#1f2328", fontsize=11)
|
|
1255
|
+
return
|
|
1256
|
+
cnt = Counter(labels).most_common(25)
|
|
1257
|
+
names, counts = zip(*cnt)
|
|
1258
|
+
y = np.arange(len(names))
|
|
1259
|
+
bars = ax.barh(y, counts, color="#58a6ff", alpha=0.85)
|
|
1260
|
+
ax.set_yticks(y)
|
|
1261
|
+
ax.set_yticklabels(names, fontsize=7)
|
|
1262
|
+
ax.set_title("Label Distribution (top 25)", color="#1f2328", fontsize=11)
|
|
1263
|
+
ax.set_xlabel("Count", color="#57606a", fontsize=8)
|
|
1264
|
+
# imbalance ratio
|
|
1265
|
+
if len(counts) > 1:
|
|
1266
|
+
ratio = max(counts) / min(counts)
|
|
1267
|
+
ax.text(0.97, 0.02, f"imbalance ratio: {ratio:.1f}×",
|
|
1268
|
+
transform=ax.transAxes, ha="right", va="bottom",
|
|
1269
|
+
fontsize=7, color="#e3b341")
|
|
1270
|
+
|
|
1271
|
+
def _plot_hist(self, ax, data, title, color, log_x=False):
|
|
1272
|
+
"""Histogram with explicit axis metrics for interpretation."""
|
|
1273
|
+
vals = np.array([v for v in data if v is not None and np.isfinite(v)])
|
|
1274
|
+
if log_x:
|
|
1275
|
+
vals = vals[vals > 0]
|
|
1276
|
+
vals = np.log10(vals + 1e-9)
|
|
1277
|
+
x_label = f"log10({title})"
|
|
1278
|
+
else:
|
|
1279
|
+
x_label = title
|
|
1280
|
+
|
|
1281
|
+
if len(vals) == 0:
|
|
1282
|
+
ax.text(0.5, 0.5, "No data", ha="center", va="center")
|
|
1283
|
+
ax.set_title(title)
|
|
1284
|
+
ax.set_xlabel(x_label)
|
|
1285
|
+
ax.set_ylabel("Number of images")
|
|
1286
|
+
return
|
|
1287
|
+
|
|
1288
|
+
ax.hist(vals, bins=30, color=color, alpha=0.85, edgecolor="white")
|
|
1289
|
+
ax.axvline(vals.mean(), color="#1f2328", ls="--", lw=1,
|
|
1290
|
+
label=f"Mean = {vals.mean():.2f}")
|
|
1291
|
+
ax.set_title(title)
|
|
1292
|
+
ax.set_xlabel(x_label)
|
|
1293
|
+
ax.set_ylabel("Number of images")
|
|
1294
|
+
ax.legend(fontsize=7)
|
|
1295
|
+
|
|
1296
|
+
def _plot_channel_histograms(self, ax, valid):
|
|
1297
|
+
means = np.array([r.mean_rgb for r in valid if r.mean_rgb])
|
|
1298
|
+
if not len(means):
|
|
1299
|
+
return
|
|
1300
|
+
colors = ["#ff6b6b", "#51cf66", "#339af0"]
|
|
1301
|
+
names = ["Red", "Green", "Blue"]
|
|
1302
|
+
for i, (c, n) in enumerate(zip(colors, names)):
|
|
1303
|
+
ax.hist(means[:, i], bins=50, color=c, alpha=0.6,
|
|
1304
|
+
label=n, edgecolor="none")
|
|
1305
|
+
ax.set_title("Per-channel Mean Distribution", color="#1f2328", fontsize=9)
|
|
1306
|
+
ax.set_xlabel("Pixel value (0–255)", color="#57606a", fontsize=8)
|
|
1307
|
+
ax.legend(fontsize=7, labelcolor="#1f2328",
|
|
1308
|
+
facecolor="#eaeef2", edgecolor="#d0d7de")
|
|
1309
|
+
|
|
1310
|
+
def _plot_dominant_colours(self, ax, valid):
|
|
1311
|
+
cv2 = _cv2()
|
|
1312
|
+
pool = [r for r in valid if not r.path.startswith("<array")][:60]
|
|
1313
|
+
all_pix = []
|
|
1314
|
+
for rec in pool:
|
|
1315
|
+
img = cv2.imread(str(rec.path))
|
|
1316
|
+
if img is None:
|
|
1317
|
+
continue
|
|
1318
|
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
|
1319
|
+
img = cv2.resize(img, (32, 32))
|
|
1320
|
+
all_pix.append(img.reshape(-1, 3))
|
|
1321
|
+
if not all_pix:
|
|
1322
|
+
return
|
|
1323
|
+
pixels = np.vstack(all_pix)
|
|
1324
|
+
try:
|
|
1325
|
+
from sklearn.cluster import MiniBatchKMeans
|
|
1326
|
+
km = MiniBatchKMeans(n_clusters=self.n_colors,
|
|
1327
|
+
random_state=42, n_init=3)
|
|
1328
|
+
if len(pixels) > 50_000:
|
|
1329
|
+
idx = np.random.choice(len(pixels), 50_000, replace=False)
|
|
1330
|
+
pixels = pixels[idx]
|
|
1331
|
+
labels = km.fit_predict(pixels)
|
|
1332
|
+
centers = km.cluster_centers_.astype(np.uint8)
|
|
1333
|
+
counts = np.bincount(labels, minlength=self.n_colors)
|
|
1334
|
+
pct = counts / counts.sum()
|
|
1335
|
+
order = np.argsort(-pct)
|
|
1336
|
+
centers, pct = centers[order], pct[order]
|
|
1337
|
+
hex_cols = [f"#{r:02x}{g:02x}{b:02x}" for r, g, b in centers]
|
|
1338
|
+
ax.barh(np.arange(len(pct)), pct * 100,
|
|
1339
|
+
color=hex_cols, edgecolor="#f6f8fa", height=0.7)
|
|
1340
|
+
ax.set_yticks(np.arange(len(pct)))
|
|
1341
|
+
ax.set_yticklabels(hex_cols, fontsize=8)
|
|
1342
|
+
ax.set_xlabel("% of pixels (sample)", color="#57606a", fontsize=8)
|
|
1343
|
+
ax.set_title("Dominant Colour Palette", color="#1f2328", fontsize=9)
|
|
1344
|
+
except Exception:
|
|
1345
|
+
pass
|
|
1346
|
+
|
|
1347
|
+
def _plot_hsv_hue_wheel(self, ax, valid):
|
|
1348
|
+
cv2 = _cv2()
|
|
1349
|
+
sample = [r for r in valid if not r.path.startswith("<array")][:100]
|
|
1350
|
+
hues = []
|
|
1351
|
+
for rec in sample:
|
|
1352
|
+
img = cv2.imread(str(rec.path))
|
|
1353
|
+
if img is None:
|
|
1354
|
+
continue
|
|
1355
|
+
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
|
|
1356
|
+
small = cv2.resize(hsv, (32, 32))
|
|
1357
|
+
hues.extend(small[:, :, 0].ravel().tolist())
|
|
1358
|
+
if not hues:
|
|
1359
|
+
return
|
|
1360
|
+
hues = np.array(hues) * 2 # OpenCV hue: 0-180 → 0-360
|
|
1361
|
+
counts, edges = np.histogram(hues, bins=36, range=(0, 360))
|
|
1362
|
+
centres = (edges[:-1] + edges[1:]) / 2
|
|
1363
|
+
colors_h = [_plt().cm.hsv(c / 360) for c in centres]
|
|
1364
|
+
ax.bar(centres, counts, width=10, color=colors_h, alpha=0.9)
|
|
1365
|
+
ax.set_title("Hue Distribution", color="#1f2328", fontsize=9)
|
|
1366
|
+
ax.set_xlabel("Hue (°)", color="#57606a", fontsize=8)
|
|
1367
|
+
|
|
1368
|
+
def _plot_saturation_value(self, ax, valid):
|
|
1369
|
+
sats = [r.mean_hsv[1] if r.mean_hsv else None for r in valid]
|
|
1370
|
+
vals = [r.mean_hsv[2] if r.mean_hsv else None for r in valid]
|
|
1371
|
+
sats = [s for s in sats if s is not None]
|
|
1372
|
+
vals = [v for v in vals if v is not None]
|
|
1373
|
+
ax.scatter(sats, vals, alpha=0.3, s=6, color="#f9c74f", linewidths=0)
|
|
1374
|
+
ax.set_xlabel("Saturation", color="#57606a", fontsize=8)
|
|
1375
|
+
ax.set_ylabel("Value (brightness)", color="#57606a", fontsize=8)
|
|
1376
|
+
ax.set_title("HSV Saturation vs Value", color="#1f2328", fontsize=9)
|
|
1377
|
+
|
|
1378
|
+
def _plot_lab_ab_scatter(self, ax, valid):
|
|
1379
|
+
labs = np.array([r.mean_lab for r in valid if r.mean_lab])
|
|
1380
|
+
if not len(labs):
|
|
1381
|
+
return
|
|
1382
|
+
ax.scatter(labs[:, 1], labs[:, 2], alpha=0.3, s=6,
|
|
1383
|
+
c=labs[:, 0], cmap="viridis", linewidths=0)
|
|
1384
|
+
ax.axhline(0, color="#d0d7de", lw=0.8)
|
|
1385
|
+
ax.axvline(0, color="#d0d7de", lw=0.8)
|
|
1386
|
+
ax.set_xlabel("a* (green–red)", color="#57606a", fontsize=8)
|
|
1387
|
+
ax.set_ylabel("b* (blue–yellow)", color="#57606a", fontsize=8)
|
|
1388
|
+
ax.set_title("Lab Colour Space (a* vs b*)", color="#1f2328", fontsize=9)
|
|
1389
|
+
|
|
1390
|
+
def _plot_color_temp_pie(self, ax, valid):
|
|
1391
|
+
ct = Counter(r.color_temp for r in valid if r.color_temp)
|
|
1392
|
+
if not ct:
|
|
1393
|
+
return
|
|
1394
|
+
colors = {"warm": "#ff6b6b", "neutral": "#57606a", "cool": "#339af0"}
|
|
1395
|
+
lbls = list(ct.keys())
|
|
1396
|
+
vals = [ct[l] for l in lbls]
|
|
1397
|
+
clrs = [colors.get(l, "#white") for l in lbls]
|
|
1398
|
+
wedges, _, autotexts = ax.pie(
|
|
1399
|
+
vals, labels=lbls, colors=clrs,
|
|
1400
|
+
autopct="%1.1f%%", startangle=90,
|
|
1401
|
+
textprops={"color": "#1f2328", "fontsize": 8},
|
|
1402
|
+
)
|
|
1403
|
+
ax.set_title("Colour Temperature", color="#1f2328", fontsize=9)
|
|
1404
|
+
|
|
1405
|
+
def _plot_texture_radar(self, ax, valid):
|
|
1406
|
+
"""Draw GLCM radar chart. ax is a plain Axes used only for position;
|
|
1407
|
+
we replace it with a polar subplot in the same grid slot."""
|
|
1408
|
+
feats = ["glcm_contrast", "glcm_homogeneity",
|
|
1409
|
+
"glcm_energy", "glcm_correlation", "glcm_asm"]
|
|
1410
|
+
labels = ["Contrast", "Homogeneity", "Energy", "Correlation", "ASM"]
|
|
1411
|
+
vals = []
|
|
1412
|
+
for f in feats:
|
|
1413
|
+
a = [getattr(r, f) for r in valid if getattr(r, f) is not None]
|
|
1414
|
+
vals.append(np.mean(a) if a else 0.0)
|
|
1415
|
+
|
|
1416
|
+
# Remove the plain axes and replace with a polar one at the same position
|
|
1417
|
+
fig = ax.get_figure()
|
|
1418
|
+
pos = ax.get_position()
|
|
1419
|
+
ax.remove()
|
|
1420
|
+
|
|
1421
|
+
if all(v == 0 for v in vals):
|
|
1422
|
+
ax2 = fig.add_axes(pos)
|
|
1423
|
+
ax2.set_facecolor("#f6f8fa")
|
|
1424
|
+
for sp in ax2.spines.values():
|
|
1425
|
+
sp.set_edgecolor("#d0d7de")
|
|
1426
|
+
ax2.text(0.5, 0.5, "GLCM not computed\n(install scikit-image)",
|
|
1427
|
+
ha="center", va="center", color="#57606a",
|
|
1428
|
+
transform=ax2.transAxes, fontsize=9)
|
|
1429
|
+
ax2.set_title("Texture Features (GLCM)", color="#1f2328", fontsize=9)
|
|
1430
|
+
ax2.set_xticks([]); ax2.set_yticks([])
|
|
1431
|
+
return
|
|
1432
|
+
|
|
1433
|
+
ax_polar = fig.add_axes(pos, projection="polar")
|
|
1434
|
+
ax_polar.set_facecolor("#f6f8fa")
|
|
1435
|
+
|
|
1436
|
+
# normalise
|
|
1437
|
+
mx = max(vals) or 1.0
|
|
1438
|
+
norm_vals = [v / mx for v in vals]
|
|
1439
|
+
norm_vals += norm_vals[:1]
|
|
1440
|
+
|
|
1441
|
+
angles = np.linspace(0, 2 * np.pi, len(labels), endpoint=False).tolist()
|
|
1442
|
+
angles += angles[:1]
|
|
1443
|
+
|
|
1444
|
+
ax_polar.set_theta_offset(np.pi / 2)
|
|
1445
|
+
ax_polar.set_theta_direction(-1)
|
|
1446
|
+
ax_polar.plot(angles, norm_vals, color="#58a6ff", lw=2)
|
|
1447
|
+
ax_polar.fill(angles, norm_vals, color="#58a6ff", alpha=0.25)
|
|
1448
|
+
ax_polar.set_xticks(angles[:-1])
|
|
1449
|
+
ax_polar.set_xticklabels(labels, color="#1f2328", fontsize=7)
|
|
1450
|
+
ax_polar.set_yticks([0.25, 0.5, 0.75, 1.0])
|
|
1451
|
+
ax_polar.set_yticklabels(["25%", "50%", "75%", "100%"],
|
|
1452
|
+
color="#57606a", fontsize=6)
|
|
1453
|
+
ax_polar.spines["polar"].set_edgecolor("#d0d7de")
|
|
1454
|
+
ax_polar.tick_params(colors="#57606a", labelsize=7)
|
|
1455
|
+
ax_polar.set_title("Texture Features (GLCM, normalised)",
|
|
1456
|
+
color="#1f2328", fontsize=9, pad=12)
|
|
1457
|
+
|
|
1458
|
+
def _plot_frequency_bands(self, ax, valid):
|
|
1459
|
+
low = [r.freq_low for r in valid if r.freq_low is not None]
|
|
1460
|
+
mid = [r.freq_mid for r in valid if r.freq_mid is not None]
|
|
1461
|
+
high = [r.freq_high for r in valid if r.freq_high is not None]
|
|
1462
|
+
if not low:
|
|
1463
|
+
ax.text(0.5, 0.5, "Frequency analysis\nnot computed",
|
|
1464
|
+
ha="center", va="center", color="#57606a",
|
|
1465
|
+
transform=ax.transAxes)
|
|
1466
|
+
ax.set_title("FFT Frequency Bands", color="#1f2328", fontsize=9)
|
|
1467
|
+
return
|
|
1468
|
+
x = np.arange(3)
|
|
1469
|
+
means = [np.mean(low), np.mean(mid), np.mean(high)]
|
|
1470
|
+
stds = [np.std(low), np.std(mid), np.std(high)]
|
|
1471
|
+
colors = ["#58a6ff", "#3fb950", "#e3b341"]
|
|
1472
|
+
ax.bar(x, means, yerr=stds, color=colors, alpha=0.85,
|
|
1473
|
+
edgecolor="none", capsize=4,
|
|
1474
|
+
error_kw=dict(ecolor="#1f2328", elinewidth=1))
|
|
1475
|
+
ax.set_xticks(x)
|
|
1476
|
+
ax.set_xticklabels(["Low\n(<10%)", "Mid\n(10–40%)", "High\n(>40%)"],
|
|
1477
|
+
color="#57606a", fontsize=8)
|
|
1478
|
+
ax.set_title("FFT Frequency Band Energy\n(mean ± std)",
|
|
1479
|
+
color="#1f2328", fontsize=9)
|
|
1480
|
+
ax.set_ylabel("Fraction of total energy", color="#57606a", fontsize=8)
|
|
1481
|
+
|
|
1482
|
+
def _plot_brightness_sharpness_scatter(self, ax, valid):
|
|
1483
|
+
b = [r.brightness for r in valid]
|
|
1484
|
+
s = [r.sharpness for r in valid]
|
|
1485
|
+
c = [r.contrast for r in valid]
|
|
1486
|
+
sc = ax.scatter(b, s, c=c, cmap="plasma", s=6,
|
|
1487
|
+
alpha=0.4, linewidths=0)
|
|
1488
|
+
ax.set_yscale("log")
|
|
1489
|
+
ax.set_xlabel("Brightness", color="#57606a", fontsize=8)
|
|
1490
|
+
ax.set_ylabel("Sharpness (log)", color="#57606a", fontsize=8)
|
|
1491
|
+
ax.set_title("Brightness vs Sharpness\n(colour = contrast)",
|
|
1492
|
+
color="#1f2328", fontsize=9)
|
|
1493
|
+
_plt().colorbar(sc, ax=ax, fraction=0.046, pad=0.04,
|
|
1494
|
+
label="Contrast").ax.tick_params(
|
|
1495
|
+
labelcolor="#57606a", labelsize=6)
|
|
1496
|
+
|
|
1497
|
+
def _plot_exposure_scatter(self, ax, valid):
|
|
1498
|
+
over = [r.overexposed_frac * 100 for r in valid]
|
|
1499
|
+
under = [r.underexposed_frac * 100 for r in valid]
|
|
1500
|
+
ax.scatter(under, over, alpha=0.3, s=6,
|
|
1501
|
+
color="#79c0ff", linewidths=0)
|
|
1502
|
+
ax.set_xlabel("Underexposed pixels (%)", color="#57606a", fontsize=8)
|
|
1503
|
+
ax.set_ylabel("Overexposed pixels (%)", color="#57606a", fontsize=8)
|
|
1504
|
+
ax.set_title("Exposure Map", color="#1f2328", fontsize=9)
|
|
1505
|
+
|
|
1506
|
+
def _plot_blurry_breakdown(self, ax, valid):
|
|
1507
|
+
sharp = sum(not r.is_blurry for r in valid)
|
|
1508
|
+
blurry = sum(r.is_blurry for r in valid)
|
|
1509
|
+
ax.bar(["Sharp", "Blurry"], [sharp, blurry],
|
|
1510
|
+
color=["#3fb950", "#f78166"], alpha=0.85, edgecolor="none")
|
|
1511
|
+
ax.set_title(f"Blurry Detection\n(threshold={self.blur_threshold})",
|
|
1512
|
+
color="#1f2328", fontsize=9)
|
|
1513
|
+
for i, v in enumerate([sharp, blurry]):
|
|
1514
|
+
ax.text(i, v + max(sharp, blurry) * 0.02, str(v),
|
|
1515
|
+
ha="center", va="bottom", color="#1f2328", fontsize=9)
|
|
1516
|
+
|
|
1517
|
+
def _plot_format_dist(self, ax, valid):
|
|
1518
|
+
cnt = Counter(r.file_ext for r in valid)
|
|
1519
|
+
if not cnt:
|
|
1520
|
+
return
|
|
1521
|
+
names, counts = zip(*cnt.most_common())
|
|
1522
|
+
colors = ["#58a6ff", "#3fb950", "#e3b341", "#f78166",
|
|
1523
|
+
"#d2a8ff", "#ffa657", "#79c0ff", "#56d364"]
|
|
1524
|
+
ax.bar(names, counts,
|
|
1525
|
+
color=colors[:len(names)], alpha=0.85, edgecolor="none")
|
|
1526
|
+
ax.set_title("File Format Distribution", color="#1f2328", fontsize=9)
|
|
1527
|
+
ax.set_xlabel("Extension", color="#57606a", fontsize=8)
|
|
1528
|
+
ax.set_ylabel("Count", color="#57606a", fontsize=8)
|
|
1529
|
+
|
|
1530
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
1531
|
+
# Helpers
|
|
1532
|
+
# ─────────────────────────────────────────────────────────────────────
|
|
1533
|
+
|
|
1534
|
+
def _resolve_paths(self, source, recursive) -> List[Path]:
|
|
1535
|
+
if isinstance(source, (list, tuple)):
|
|
1536
|
+
return [Path(p) for p in source]
|
|
1537
|
+
source = Path(source)
|
|
1538
|
+
if source.is_file():
|
|
1539
|
+
return [source]
|
|
1540
|
+
pattern = "**/*" if recursive else "*"
|
|
1541
|
+
return sorted(
|
|
1542
|
+
p for p in source.glob(pattern)
|
|
1543
|
+
if p.is_file() and p.suffix.lower() in self.SUPPORTED_EXTS
|
|
1544
|
+
)
|
|
1545
|
+
|
|
1546
|
+
@staticmethod
|
|
1547
|
+
def _to_uint8_rgb(arr: np.ndarray) -> np.ndarray:
|
|
1548
|
+
if arr.dtype in (np.float32, np.float64):
|
|
1549
|
+
arr = (np.clip(arr, 0, 1) * 255).astype(np.uint8)
|
|
1550
|
+
if arr.ndim == 2:
|
|
1551
|
+
arr = np.stack([arr, arr, arr], axis=-1)
|
|
1552
|
+
elif arr.shape[2] == 4:
|
|
1553
|
+
arr = arr[:, :, :3]
|
|
1554
|
+
return arr.astype(np.uint8)
|
|
1555
|
+
|
|
1556
|
+
def _log(self, msg: str) -> None:
|
|
1557
|
+
if self.verbose:
|
|
1558
|
+
print(f"[viseda] {msg}")
|
|
1559
|
+
|
|
1560
|
+
def _check_loaded(self):
|
|
1561
|
+
if not self._loaded:
|
|
1562
|
+
raise RuntimeError("Call .load() or .load_arrays() before analysing.")
|
|
1563
|
+
|
|
1564
|
+
@staticmethod
|
|
1565
|
+
def _finalise(fig, save_path, dpi):
|
|
1566
|
+
plt = _plt()
|
|
1567
|
+
if save_path:
|
|
1568
|
+
fig.savefig(save_path, dpi=dpi, bbox_inches="tight",
|
|
1569
|
+
facecolor=fig.get_facecolor())
|
|
1570
|
+
else:
|
|
1571
|
+
plt.show()
|
|
1572
|
+
plt.close(fig)
|
|
1573
|
+
|
|
1574
|
+
|
|
1575
|
+
# ═════════════════════════════════════════════════════════════════════════════
|
|
1576
|
+
# Utilities
|
|
1577
|
+
# ═════════════════════════════════════════════════════════════════════════════
|
|
1578
|
+
|
|
1579
|
+
def _stat_dict(arr) -> Dict[str, float]:
|
|
1580
|
+
arr = np.asarray([x for x in arr if x is not None and np.isfinite(x)])
|
|
1581
|
+
if len(arr) == 0:
|
|
1582
|
+
return {}
|
|
1583
|
+
return {
|
|
1584
|
+
"min": round(float(np.min(arr)), 4),
|
|
1585
|
+
"max": round(float(np.max(arr)), 4),
|
|
1586
|
+
"mean": round(float(np.mean(arr)), 4),
|
|
1587
|
+
"median": round(float(np.median(arr)), 4),
|
|
1588
|
+
"std": round(float(np.std(arr)), 4),
|
|
1589
|
+
"p25": round(float(np.percentile(arr, 25)), 4),
|
|
1590
|
+
"p75": round(float(np.percentile(arr, 75)), 4),
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
|
|
1594
|
+
def _resize_display(img: np.ndarray, max_side: int) -> np.ndarray:
|
|
1595
|
+
cv2 = _cv2()
|
|
1596
|
+
h, w = img.shape[:2]
|
|
1597
|
+
scale = min(max_side / h, max_side / w, 1.0)
|
|
1598
|
+
if scale < 1.0:
|
|
1599
|
+
img = cv2.resize(img, (int(w * scale), int(h * scale)),
|
|
1600
|
+
interpolation=cv2.INTER_AREA)
|
|
1601
|
+
return img
|
|
1602
|
+
|
|
1603
|
+
|
|
1604
|
+
# ═════════════════════════════════════════════════════════════════════════════
|
|
1605
|
+
# HTML Report
|
|
1606
|
+
# ═════════════════════════════════════════════════════════════════════════════
|
|
1607
|
+
|
|
1608
|
+
def _generate_html_report(summary: Dict[str, Any], output_path: str) -> None:
|
|
1609
|
+
inv = summary.get("inventory", {})
|
|
1610
|
+
sp = summary.get("spatial", {})
|
|
1611
|
+
px = summary.get("pixel_stats", {})
|
|
1612
|
+
qu = summary.get("quality", {})
|
|
1613
|
+
co = summary.get("colour", {})
|
|
1614
|
+
tx = summary.get("texture", {})
|
|
1615
|
+
fr = summary.get("frequency", {})
|
|
1616
|
+
du = summary.get("duplicates",{})
|
|
1617
|
+
lb = summary.get("labels", {})
|
|
1618
|
+
nm = summary.get("normalization_stats", {})
|
|
1619
|
+
|
|
1620
|
+
def card(title, stats):
|
|
1621
|
+
if not stats:
|
|
1622
|
+
return ""
|
|
1623
|
+
rows = "".join(
|
|
1624
|
+
f'<div class="stat"><span>{k}</span>'
|
|
1625
|
+
f'<span class="val">{_fmt(v)}</span></div>'
|
|
1626
|
+
for k, v in stats.items()
|
|
1627
|
+
)
|
|
1628
|
+
return f'<div class="card"><h3>{title}</h3>{rows}</div>'
|
|
1629
|
+
|
|
1630
|
+
def badge(text, cls="blue"):
|
|
1631
|
+
return f'<span class="badge badge-{cls}">{text}</span>'
|
|
1632
|
+
|
|
1633
|
+
def bar_chart(title, dist, span=1):
|
|
1634
|
+
if not dist:
|
|
1635
|
+
return ""
|
|
1636
|
+
total = sum(dist.values()) or 1
|
|
1637
|
+
mx = max(dist.values()) or 1
|
|
1638
|
+
rows = ""
|
|
1639
|
+
for lbl, cnt in sorted(dist.items(), key=lambda x: -x[1])[:25]:
|
|
1640
|
+
pct = cnt / mx * 100
|
|
1641
|
+
rows += (
|
|
1642
|
+
f'<div class="bar-row">'
|
|
1643
|
+
f'<span class="bar-label" title="{lbl}">{lbl}</span>'
|
|
1644
|
+
f'<div class="bar">'
|
|
1645
|
+
f'<div class="bar-fill" style="width:{pct:.1f}%"></div></div>'
|
|
1646
|
+
f'<span class="bar-count">{cnt:,}</span></div>'
|
|
1647
|
+
)
|
|
1648
|
+
span_style = f'grid-column: span {span};' if span > 1 else ''
|
|
1649
|
+
return (f'<div class="card" style="{span_style}">'
|
|
1650
|
+
f'<h3>{title}</h3><div class="bar-wrap">{rows}</div></div>')
|
|
1651
|
+
|
|
1652
|
+
badges_html = (
|
|
1653
|
+
badge(f"{inv.get('total', 0):,} images") +
|
|
1654
|
+
badge(f"{inv.get('valid', 0):,} valid", "green") +
|
|
1655
|
+
(badge(f"{inv.get('corrupt', 0):,} corrupt", "red")
|
|
1656
|
+
if inv.get("corrupt") else "") +
|
|
1657
|
+
badge(f"{du.get('n_exact_duplicate_groups', 0)} exact dupes", "yellow") +
|
|
1658
|
+
badge(f"{du.get('n_near_duplicate_groups', 0)} near dupes", "yellow")
|
|
1659
|
+
)
|
|
1660
|
+
|
|
1661
|
+
norm_html = ""
|
|
1662
|
+
if nm:
|
|
1663
|
+
m = [f"{v:.4f}" for v in nm.get("mean", [])]
|
|
1664
|
+
s = [f"{v:.4f}" for v in nm.get("std", [])]
|
|
1665
|
+
norm_html = f"""
|
|
1666
|
+
<h2>📐 Normalisation Stats (for torchvision / transforms)</h2>
|
|
1667
|
+
<div class="card"><pre style="color:#58a6ff;font-size:0.85rem;">
|
|
1668
|
+
transforms.Normalize(
|
|
1669
|
+
mean={m},
|
|
1670
|
+
std ={s}
|
|
1671
|
+
)</pre></div>"""
|
|
1672
|
+
|
|
1673
|
+
corrupt_html = ""
|
|
1674
|
+
if inv.get("corrupt_paths"):
|
|
1675
|
+
items = "".join(f"<li>{p}</li>"
|
|
1676
|
+
for p in inv["corrupt_paths"][:50])
|
|
1677
|
+
corrupt_html = (f'<h2>⚠️ Corrupt Files</h2>'
|
|
1678
|
+
f'<div class="card corrupt"><ul>{items}</ul></div>')
|
|
1679
|
+
|
|
1680
|
+
# Pre-build dicts that can't go inside f-string {{ }} literals
|
|
1681
|
+
blurry_card = card("Blurry Summary", {
|
|
1682
|
+
"Blurry count": qu.get("blurry_count"),
|
|
1683
|
+
"Blurry fraction": qu.get("blurry_fraction"),
|
|
1684
|
+
})
|
|
1685
|
+
greyscale_card = card("Greyscale-like", {
|
|
1686
|
+
"Count": co.get("grayscale_like_count"),
|
|
1687
|
+
"Fraction": co.get("grayscale_like_fraction"),
|
|
1688
|
+
})
|
|
1689
|
+
duplicate_card = card("Duplicate Summary", {
|
|
1690
|
+
"Exact duplicate groups": du.get("n_exact_duplicate_groups"),
|
|
1691
|
+
"Near duplicate groups": du.get("n_near_duplicate_groups"),
|
|
1692
|
+
})
|
|
1693
|
+
class_imb_card = card("Class Imbalance", {
|
|
1694
|
+
"Imbalance ratio (max/min)": lb.get("class_imbalance_ratio"),
|
|
1695
|
+
})
|
|
1696
|
+
counts_card = card("Counts", {
|
|
1697
|
+
"Total": inv.get("total"),
|
|
1698
|
+
"Valid": inv.get("valid"),
|
|
1699
|
+
"Corrupt": inv.get("corrupt"),
|
|
1700
|
+
})
|
|
1701
|
+
mean_rgb_card = card("Dataset Mean RGB (0-255)",
|
|
1702
|
+
dict(zip(["R mean", "G mean", "B mean"],
|
|
1703
|
+
[round(v, 3) for v in px.get("dataset_mean_rgb", [])])))
|
|
1704
|
+
std_rgb_card = card("Dataset Pixel Std RGB",
|
|
1705
|
+
dict(zip(["R std", "G std", "B std"],
|
|
1706
|
+
[round(v, 3) for v in px.get("dataset_pixel_std_rgb", [])])))
|
|
1707
|
+
|
|
1708
|
+
html = f"""<!DOCTYPE html>
|
|
1709
|
+
<html lang="en"><head><meta charset="UTF-8"/>
|
|
1710
|
+
<meta name="viewport" content="width=device-width,initial-scale=1"/>
|
|
1711
|
+
<title>VisEDA Report</title>
|
|
1712
|
+
<style>
|
|
1713
|
+
:root{{--bg:white;--surface:#f6f8fa;--border:#d0d7de;--text:#1f2328;
|
|
1714
|
+
--muted:#57606a;--accent:#58a6ff;--green:#3fb950;--red:#f78166;
|
|
1715
|
+
--yellow:#e3b341;}}
|
|
1716
|
+
*{{box-sizing:border-box;margin:0;padding:0}}
|
|
1717
|
+
body{{background:var(--bg);color:var(--text);
|
|
1718
|
+
font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;
|
|
1719
|
+
padding:2rem;}}
|
|
1720
|
+
h1{{font-size:1.9rem;margin-bottom:.25rem}}
|
|
1721
|
+
h2{{font-size:1.05rem;color:var(--accent);margin:1.8rem 0 .6rem;}}
|
|
1722
|
+
h3{{font-size:.78rem;color:var(--muted);text-transform:uppercase;
|
|
1723
|
+
letter-spacing:.05em;margin-bottom:.5rem}}
|
|
1724
|
+
.sub{{color:var(--muted);font-size:.85rem;margin-bottom:1.5rem}}
|
|
1725
|
+
.grid{{display:grid;grid-template-columns:repeat(auto-fill,minmax(240px,1fr));
|
|
1726
|
+
gap:.9rem}}
|
|
1727
|
+
.card{{background:var(--surface);border:1px solid var(--border);
|
|
1728
|
+
border-radius:8px;padding:1rem}}
|
|
1729
|
+
.stat{{display:flex;justify-content:space-between;font-size:.82rem;
|
|
1730
|
+
padding:.18rem 0;border-bottom:1px solid var(--border)}}
|
|
1731
|
+
.stat:last-child{{border-bottom:none}}
|
|
1732
|
+
.val{{color:var(--accent);font-variant-numeric:tabular-nums}}
|
|
1733
|
+
.badge{{display:inline-block;padding:.15rem .5rem;border-radius:12px;
|
|
1734
|
+
font-size:.72rem;font-weight:600;margin:.15rem}}
|
|
1735
|
+
.badge-blue{{background:rgba(88,166,255,.15);color:var(--accent)}}
|
|
1736
|
+
.badge-green{{background:rgba(63,185,80,.15);color:var(--green)}}
|
|
1737
|
+
.badge-red{{background:rgba(247,129,102,.15);color:var(--red)}}
|
|
1738
|
+
.badge-yellow{{background:rgba(227,179,65,.15);color:var(--yellow)}}
|
|
1739
|
+
.bar-wrap{{margin-top:.4rem}}
|
|
1740
|
+
.bar-row{{display:flex;align-items:center;gap:.4rem;margin:.18rem 0;
|
|
1741
|
+
font-size:.76rem}}
|
|
1742
|
+
.bar-label{{width:120px;overflow:hidden;text-overflow:ellipsis;
|
|
1743
|
+
white-space:nowrap;color:var(--muted)}}
|
|
1744
|
+
.bar{{flex:1;background:var(--border);border-radius:3px;height:9px}}
|
|
1745
|
+
.bar-fill{{height:100%;border-radius:3px;background:var(--accent)}}
|
|
1746
|
+
.bar-count{{width:55px;text-align:right;color:var(--accent)}}
|
|
1747
|
+
.corrupt{{max-height:200px;overflow-y:auto;font-size:.78rem;
|
|
1748
|
+
color:var(--red)}}
|
|
1749
|
+
pre{{background:#eaeef2;padding:.8rem;border-radius:6px;
|
|
1750
|
+
overflow-x:auto;font-size:.82rem}}
|
|
1751
|
+
footer{{margin-top:3rem;color:var(--muted);font-size:.72rem;
|
|
1752
|
+
border-top:1px solid var(--border);padding-top:1rem}}
|
|
1753
|
+
</style></head><body>
|
|
1754
|
+
<h1>🔬 VisEDA — Image EDA Report</h1>
|
|
1755
|
+
<p class="sub">Generated by <strong>VisEDA</strong></p>
|
|
1756
|
+
<p style="margin-bottom:1rem">{badges_html}</p>
|
|
1757
|
+
|
|
1758
|
+
<h2>📦 Inventory</h2>
|
|
1759
|
+
<div class="grid">
|
|
1760
|
+
{counts_card}
|
|
1761
|
+
{bar_chart("Format Distribution", inv.get("format_distribution", {}))}
|
|
1762
|
+
{bar_chart("Colour Mode", inv.get("colour_mode_distribution", {}))}
|
|
1763
|
+
</div>
|
|
1764
|
+
|
|
1765
|
+
<h2>📐 Spatial</h2>
|
|
1766
|
+
<div class="grid">
|
|
1767
|
+
{card("Height (px)", sp.get("height", {}))}
|
|
1768
|
+
{card("Width (px)", sp.get("width", {}))}
|
|
1769
|
+
{card("Aspect Ratio", sp.get("aspect_ratio", {}))}
|
|
1770
|
+
{card("Megapixels", sp.get("megapixels", {}))}
|
|
1771
|
+
{card("File Size (KB)", sp.get("file_size_kb", {}))}
|
|
1772
|
+
{bar_chart("Orientation", sp.get("orientation_distribution", {}))}
|
|
1773
|
+
</div>
|
|
1774
|
+
|
|
1775
|
+
<h2>🎨 Pixel Statistics</h2>
|
|
1776
|
+
<div class="grid">
|
|
1777
|
+
{mean_rgb_card}
|
|
1778
|
+
{std_rgb_card}
|
|
1779
|
+
</div>
|
|
1780
|
+
|
|
1781
|
+
{norm_html}
|
|
1782
|
+
|
|
1783
|
+
<h2>🔍 Quality Metrics</h2>
|
|
1784
|
+
<div class="grid">
|
|
1785
|
+
{card("Brightness", qu.get("brightness", {}))}
|
|
1786
|
+
{card("Contrast", qu.get("contrast", {}))}
|
|
1787
|
+
{card("Sharpness", qu.get("sharpness", {}))}
|
|
1788
|
+
{card("Noise Estimate", qu.get("noise_estimate", {}))}
|
|
1789
|
+
{card("Pixel Entropy", qu.get("entropy", {}))}
|
|
1790
|
+
{card("Compression Score", qu.get("compression_score",{}))}
|
|
1791
|
+
{card("Overexposed Frac", qu.get("overexposed_frac", {}))}
|
|
1792
|
+
{card("Underexposed Frac", qu.get("underexposed_frac",{}))}
|
|
1793
|
+
{blurry_card}
|
|
1794
|
+
</div>
|
|
1795
|
+
|
|
1796
|
+
<h2>🌈 Colour</h2>
|
|
1797
|
+
<div class="grid">
|
|
1798
|
+
{card("Saturation (HSV-S)", co.get("saturation", {}))}
|
|
1799
|
+
{bar_chart("Colour Temperature", co.get("colour_temp_distribution", {}))}
|
|
1800
|
+
{greyscale_card}
|
|
1801
|
+
</div>
|
|
1802
|
+
|
|
1803
|
+
<h2>🧱 Texture (GLCM)</h2>
|
|
1804
|
+
<div class="grid">
|
|
1805
|
+
{"".join(card(k.replace("_"," ").title(), v) for k, v in tx.items())}
|
|
1806
|
+
</div>
|
|
1807
|
+
|
|
1808
|
+
<h2>〰️ Frequency (FFT)</h2>
|
|
1809
|
+
<div class="grid">
|
|
1810
|
+
{"".join(card(k.replace("_"," ").title(), v) for k, v in fr.items())}
|
|
1811
|
+
</div>
|
|
1812
|
+
|
|
1813
|
+
<h2>🔁 Duplicates</h2>
|
|
1814
|
+
<div class="grid">
|
|
1815
|
+
{duplicate_card}
|
|
1816
|
+
</div>
|
|
1817
|
+
|
|
1818
|
+
<h2>🏷️ Labels</h2>
|
|
1819
|
+
<div class="grid">
|
|
1820
|
+
{bar_chart("Label Distribution",
|
|
1821
|
+
lb.get("label_distribution") or {}, span=2)}
|
|
1822
|
+
{class_imb_card}
|
|
1823
|
+
</div>
|
|
1824
|
+
|
|
1825
|
+
{corrupt_html}
|
|
1826
|
+
|
|
1827
|
+
<footer>Generated by VisEDA — Visual Exploratory Data Analysis</footer>
|
|
1828
|
+
</body></html>"""
|
|
1829
|
+
|
|
1830
|
+
Path(output_path).write_text(html, encoding="utf-8")
|
|
1831
|
+
|
|
1832
|
+
|
|
1833
|
+
def _fmt(v) -> str:
|
|
1834
|
+
if v is None:
|
|
1835
|
+
return "N/A"
|
|
1836
|
+
if isinstance(v, float):
|
|
1837
|
+
return f"{v:,.4f}"
|
|
1838
|
+
if isinstance(v, int):
|
|
1839
|
+
return f"{v:,}"
|
|
1840
|
+
return str(v)
|