dset-cli 0.1.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dset-cli
3
- Version: 0.1.0
3
+ Version: 0.2.1
4
4
  Summary: Version control for datasets — commit, diff, and roll back millions of files with semantic diffs.
5
5
  Requires-Python: >=3.9
6
6
  Description-Content-Type: text/markdown
7
+ Requires-Dist: Pillow>=9.0
7
8
 
8
9
  # dset — version control for datasets
9
10
 
@@ -69,6 +70,7 @@ dset checkout v1
69
70
  | `dset log` | list versions with file counts and sizes |
70
71
  | `dset diff <a> <b>` | semantic diff: file counts by type, size, class distribution shift |
71
72
  | `dset checkout <ref>` | restore the working tree to a version (`--force` to discard changes) |
73
+ | `dset check [path]` | quality scan: duplicates, corruption, leakage, imbalance, outliers (`--fix` to quarantine) |
72
74
 
73
75
  Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
74
76
 
@@ -88,6 +90,28 @@ Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
88
90
  corrupt history. On btrfs/XFS the copy is a free copy-on-write clone.
89
91
  - Uncommitted changes block `dset checkout` unless you pass `--force`.
90
92
 
93
+ ## dset check — the quality engine
94
+
95
+ ```
96
+ dset check ./my-dataset # scan and score
97
+ dset check ./my-dataset --fix # also quarantine exact dups + corrupted
98
+ ```
99
+
100
+ Scans every image (parallel, ~2,000 images/sec) and reports:
101
+
102
+ - corrupted / unreadable images
103
+ - exact duplicates (content hash) and near-duplicates (perceptual dhash + LSH)
104
+ - train/test leakage — exact and near-duplicate pairs across train/ and test/
105
+ - class imbalance (parsed from CSV/TSV label files)
106
+ - outliers: anomalous file size, odd resolutions, near-blank images
107
+ - images missing from label files, and label rows pointing to missing files
108
+
109
+ Prints a health score out of 100 and writes a browsable `dset_report.html`.
110
+ `--fix` never deletes anything — it moves exact duplicates and corrupted
111
+ files to `_dset_quarantine/` for you to review.
112
+
113
+ Requires Pillow (installed automatically with the package).
114
+
91
115
  ## Current limits (MVP)
92
116
 
93
117
  - Class stats read CSV/TSV label files only (COCO/YOLO parsers are next).
@@ -62,6 +62,7 @@ dset checkout v1
62
62
  | `dset log` | list versions with file counts and sizes |
63
63
  | `dset diff <a> <b>` | semantic diff: file counts by type, size, class distribution shift |
64
64
  | `dset checkout <ref>` | restore the working tree to a version (`--force` to discard changes) |
65
+ | `dset check [path]` | quality scan: duplicates, corruption, leakage, imbalance, outliers (`--fix` to quarantine) |
65
66
 
66
67
  Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
67
68
 
@@ -81,6 +82,28 @@ Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
81
82
  corrupt history. On btrfs/XFS the copy is a free copy-on-write clone.
82
83
  - Uncommitted changes block `dset checkout` unless you pass `--force`.
83
84
 
85
+ ## dset check — the quality engine
86
+
87
+ ```
88
+ dset check ./my-dataset # scan and score
89
+ dset check ./my-dataset --fix # also quarantine exact dups + corrupted
90
+ ```
91
+
92
+ Scans every image (parallel, ~2,000 images/sec) and reports:
93
+
94
+ - corrupted / unreadable images
95
+ - exact duplicates (content hash) and near-duplicates (perceptual dhash + LSH)
96
+ - train/test leakage — exact and near-duplicate pairs across train/ and test/
97
+ - class imbalance (parsed from CSV/TSV label files)
98
+ - outliers: anomalous file size, odd resolutions, near-blank images
99
+ - images missing from label files, and label rows pointing to missing files
100
+
101
+ Prints a health score out of 100 and writes a browsable `dset_report.html`.
102
+ `--fix` never deletes anything — it moves exact duplicates and corrupted
103
+ files to `_dset_quarantine/` for you to review.
104
+
105
+ Requires Pillow (installed automatically with the package).
106
+
84
107
  ## Current limits (MVP)
85
108
 
86
109
  - Class stats read CSV/TSV label files only (COCO/YOLO parsers are next).
@@ -0,0 +1,388 @@
1
+ """dset check — the dataset quality engine.
2
+
3
+ Scans a folder of images + label files and reports:
4
+ corrupted images, exact duplicates, near-duplicates, class imbalance,
5
+ train/test leakage, outliers (size / resolution / brightness),
6
+ and label <-> file mismatches.
7
+
8
+ Writes a dset_report.html and prints a health score.
9
+ `--fix` moves exact duplicates and corrupted files into _dset_quarantine/.
10
+
11
+ Requires Pillow (pip install Pillow). Everything else is stdlib.
12
+ """
13
+
14
+ import csv
15
+ import hashlib
16
+ import html
17
+ import json
18
+ import os
19
+ import shutil
20
+ import statistics
21
+ import sys
22
+ import time
23
+ from concurrent.futures import ThreadPoolExecutor
24
+ from pathlib import Path
25
+
26
+ IMAGE_EXT = {".jpg", ".jpeg", ".png", ".tif", ".tiff", ".bmp", ".gif", ".webp"}
27
+ CLASS_COLUMNS = ("label", "class", "class_name", "category", "label_name", "target")
28
+ FILE_COLUMNS = ("file", "filename", "path", "image", "id", "name")
29
+
30
+
31
+ def _col(s, c, tty):
32
+ codes = {"g": "32", "r": "31", "y": "33", "d": "90", "b": "1"}
33
+ return f"\033[{codes[c]}m{s}\033[0m" if tty else str(s)
34
+
35
+
36
+ # ---------------------------------------------------------------- per-image
37
+
38
+ def _analyze_image(path):
39
+ """Hash + decode one image. Returns a record dict; ok=False if corrupt."""
40
+ from PIL import Image
41
+ rec = {"path": str(path), "bytes": 0, "sha": None, "ok": False,
42
+ "w": 0, "h": 0, "dhash": None, "bright": None}
43
+ try:
44
+ data = path.read_bytes()
45
+ except OSError:
46
+ return rec
47
+ rec["bytes"] = len(data)
48
+ rec["sha"] = hashlib.sha256(data).hexdigest()
49
+ try:
50
+ with Image.open(path) as im:
51
+ im.load()
52
+ rec["w"], rec["h"] = im.size
53
+ g = im.convert("L").resize((9, 8))
54
+ px = list(g.getdata())
55
+ bits = 0
56
+ for r in range(8):
57
+ for c in range(8):
58
+ bits = (bits << 1) | (px[r * 9 + c] > px[r * 9 + c + 1])
59
+ rec["dhash"] = bits
60
+ rec["bright"] = sum(px) / len(px)
61
+ rec["ok"] = True
62
+ except Exception:
63
+ pass
64
+ return rec
65
+
66
+
67
+ def _hamming(a, b):
68
+ return bin(a ^ b).count("1")
69
+
70
+
71
+ def _near_dup_groups(recs, max_dist=6):
72
+ """LSH banding on the 64-bit dhash: candidates share one 16-bit band."""
73
+ buckets = {}
74
+ for i, r in enumerate(recs):
75
+ hsh = r["dhash"]
76
+ for band in range(4):
77
+ key = (band, (hsh >> (band * 16)) & 0xFFFF)
78
+ buckets.setdefault(key, []).append(i)
79
+ parent = list(range(len(recs)))
80
+
81
+ def find(x):
82
+ while parent[x] != x:
83
+ parent[x] = parent[parent[x]]
84
+ x = parent[x]
85
+ return x
86
+
87
+ checked = set()
88
+ for idxs in buckets.values():
89
+ if len(idxs) < 2 or len(idxs) > 400: # skip degenerate mega-buckets
90
+ continue
91
+ for i in range(len(idxs)):
92
+ for j in range(i + 1, len(idxs)):
93
+ a, b = idxs[i], idxs[j]
94
+ if (a, b) in checked:
95
+ continue
96
+ checked.add((a, b))
97
+ if _hamming(recs[a]["dhash"], recs[b]["dhash"]) <= max_dist:
98
+ parent[find(a)] = find(b)
99
+ groups = {}
100
+ for i in range(len(recs)):
101
+ groups.setdefault(find(i), []).append(i)
102
+ return [g for g in groups.values() if len(g) > 1]
103
+
104
+
105
+ # ---------------------------------------------------------------- labels
106
+
107
+ def _read_labels(root):
108
+ """Parse CSV/TSV label files. Returns (class_counts, file->class, sources)."""
109
+ counts, file_map, sources = {}, {}, []
110
+ for p in sorted(root.rglob("*")):
111
+ if p.suffix.lower() not in (".csv", ".tsv") or not p.is_file():
112
+ continue
113
+ if any(part.startswith((".", "_dset")) for part in p.parts):
114
+ continue
115
+ delim = "\t" if p.suffix.lower() == ".tsv" else ","
116
+ try:
117
+ with open(p, newline="", encoding="utf-8", errors="replace") as f:
118
+ reader = csv.reader(f, delimiter=delim)
119
+ header = next(reader, None)
120
+ if not header:
121
+ continue
122
+ lower = [h.strip().lower() for h in header]
123
+ ci = next((lower.index(c) for c in CLASS_COLUMNS if c in lower), None)
124
+ fi = next((lower.index(c) for c in FILE_COLUMNS if c in lower), None)
125
+ if ci is None:
126
+ continue
127
+ sources.append(p)
128
+ for row in reader:
129
+ if ci < len(row) and row[ci].strip():
130
+ v = row[ci].strip()
131
+ counts[v] = counts.get(v, 0) + 1
132
+ if fi is not None and fi < len(row):
133
+ file_map[row[fi].strip()] = v
134
+ except OSError:
135
+ continue
136
+ return counts, file_map, sources
137
+
138
+
139
+ # ---------------------------------------------------------------- report
140
+
141
+ def _write_html(out, ctx):
142
+ rows = []
143
+ for sev, text, detail in ctx["findings"]:
144
+ color = {"ok": "#0a7d33", "warn": "#b0771e", "bad": "#c0392b"}[sev]
145
+ mark = {"ok": "✓", "warn": "⚠", "bad": "✕"}[sev]
146
+ d = f'<div class="detail">{html.escape(detail)}</div>' if detail else ""
147
+ rows.append(f'<div class="finding"><span style="color:{color}">{mark}</span>'
148
+ f'<div>{html.escape(text)}{d}</div></div>')
149
+
150
+ def listing(title, items):
151
+ if not items:
152
+ return ""
153
+ lis = "".join(f"<li>{html.escape(i)}</li>" for i in items[:60])
154
+ more = (f'<p class="more">… and {len(items) - 60:,} more</p>'
155
+ if len(items) > 60 else "")
156
+ return (f'<details><summary>{html.escape(title)} ({len(items):,})</summary>'
157
+ f"<ul>{lis}</ul>{more}</details>")
158
+
159
+ score = ctx["score"]
160
+ color = "#0a7d33" if score >= 85 else "#b0771e" if score >= 60 else "#c0392b"
161
+ doc = f"""<!DOCTYPE html><html><head><meta charset="utf-8">
162
+ <title>dset report — {html.escape(ctx['root'])}</title><style>
163
+ body{{font-family:system-ui,sans-serif;max-width:760px;margin:40px auto;
164
+ padding:0 24px;color:#111;line-height:1.6}}
165
+ h1{{font-size:20px;font-weight:600}} .score{{font-size:52px;font-weight:600;
166
+ color:{color};margin:8px 0 4px}} .meta{{color:#777;font-size:14px}}
167
+ .finding{{display:flex;gap:12px;padding:10px 0;border-bottom:1px solid #eee;
168
+ font-size:15px}} .detail{{color:#777;font-size:13px}}
169
+ details{{margin:14px 0;font-size:14px}} summary{{cursor:pointer;font-weight:500}}
170
+ ul{{color:#555;font-family:ui-monospace,monospace;font-size:12.5px}}
171
+ .more{{color:#999;font-size:13px}}</style></head><body>
172
+ <h1>dset check — {html.escape(ctx['root'])}</h1>
173
+ <div class="score">{score} / 100</div>
174
+ <div class="meta">{ctx['n_images']:,} images · {ctx['n_files']:,} files scanned ·
175
+ {time.strftime('%Y-%m-%d %H:%M')}</div><br>
176
+ {''.join(rows)}
177
+ {listing('Corrupted files', ctx['corrupted'])}
178
+ {listing('Exact duplicate files (kept one per group)', ctx['dup_files'])}
179
+ {listing('Near-duplicate groups (first file of each)', ctx['near_files'])}
180
+ {listing('Suspected train/test leakage pairs', ctx['leak_pairs'])}
181
+ {listing('Outlier images', ctx['outlier_files'])}
182
+ {listing('Images missing from label files', ctx['unlabeled'])}
183
+ {listing('Label rows pointing to missing files', ctx['orphan_labels'])}
184
+ </body></html>"""
185
+ Path(out).write_text(doc)
186
+
187
+
188
+ # ---------------------------------------------------------------- command
189
+
190
+ def run_check(root_arg, fix=False, report_path=None):
191
+ try:
192
+ import PIL # noqa: F401
193
+ except ImportError:
194
+ print("dset check needs Pillow: pip install Pillow", file=sys.stderr)
195
+ sys.exit(1)
196
+
197
+ tty = sys.stdout.isatty() and os.environ.get("NO_COLOR") is None
198
+ root = Path(root_arg).resolve()
199
+ if not root.exists():
200
+ print(f"error: path not found: {root}", file=sys.stderr)
201
+ sys.exit(1)
202
+
203
+ paths = [p for p in sorted(root.rglob("*"))
204
+ if p.is_file()
205
+ and not any(part.startswith((".", "_dset")) for part in p.parts)]
206
+ images = [p for p in paths if p.suffix.lower() in IMAGE_EXT]
207
+ print(f"Analyzing {len(images):,} images ({len(paths):,} files) in {root} ...")
208
+
209
+ t0 = time.time()
210
+ recs = []
211
+ with ThreadPoolExecutor(max_workers=min(32, (os.cpu_count() or 4) * 2)) as ex:
212
+ for i, rec in enumerate(ex.map(_analyze_image, images), 1):
213
+ recs.append(rec)
214
+ if i % 2000 == 0:
215
+ print(f"\r {i:,}/{len(images):,} ...", end="", flush=True)
216
+ print("\r" + " " * 30 + "\r", end="")
217
+
218
+ ok = [r for r in recs if r["ok"]]
219
+ corrupted = [r["path"] for r in recs if not r["ok"]]
220
+
221
+ # exact duplicates (by content hash)
222
+ by_sha = {}
223
+ for r in ok:
224
+ by_sha.setdefault(r["sha"], []).append(r["path"])
225
+ dup_groups = [v for v in by_sha.values() if len(v) > 1]
226
+ dup_files = [p for g in dup_groups for p in g[1:]] # all but first
227
+
228
+ # near duplicates (exclude exact dups so groups aren't double-counted)
229
+ firsts = {g[0] for g in dup_groups}
230
+ uniq = [r for r in ok if r["sha"] in {k for k, v in by_sha.items()}
231
+ and (len(by_sha[r["sha"]]) == 1 or r["path"] in firsts)]
232
+ near_groups_idx = _near_dup_groups(uniq)
233
+ near_files = [uniq[g[0]]["path"] for g in near_groups_idx]
234
+ n_near = sum(len(g) for g in near_groups_idx) - len(near_groups_idx)
235
+
236
+ # train/test leakage
237
+ def split_of(p):
238
+ parts = [x.lower() for x in Path(p).parts]
239
+ if any("train" in x for x in parts):
240
+ return "train"
241
+ if any(("test" in x or "val" in x) for x in parts):
242
+ return "test"
243
+ return None
244
+
245
+ leak_pairs = []
246
+ tr = {r["sha"]: r["path"] for r in ok if split_of(r["path"]) == "train"}
247
+ te = [r for r in ok if split_of(r["path"]) == "test"]
248
+ for r in te:
249
+ if r["sha"] in tr:
250
+ leak_pairs.append(f"{tr[r['sha']]} == {r['path']}")
251
+ if te and tr:
252
+ tr_hashes = {}
253
+ for r in ok:
254
+ if split_of(r["path"]) == "train":
255
+ for band in range(4):
256
+ key = (band, (r["dhash"] >> (band * 16)) & 0xFFFF)
257
+ tr_hashes.setdefault(key, []).append(r)
258
+ seen = set()
259
+ for r in te:
260
+ for band in range(4):
261
+ key = (band, (r["dhash"] >> (band * 16)) & 0xFFFF)
262
+ for cand in tr_hashes.get(key, [])[:200]:
263
+ pair = (cand["path"], r["path"])
264
+ if pair in seen or cand["sha"] == r["sha"]:
265
+ continue
266
+ seen.add(pair)
267
+ if _hamming(cand["dhash"], r["dhash"]) <= 6:
268
+ leak_pairs.append(f"{cand['path']} ≈ {r['path']}")
269
+
270
+ # outliers: file size, resolution, brightness
271
+ outlier_files = []
272
+ if len(ok) >= 20:
273
+ sizes = [r["bytes"] for r in ok]
274
+ mu, sd = statistics.mean(sizes), statistics.pstdev(sizes) or 1
275
+ res = {}
276
+ for r in ok:
277
+ res[(r["w"], r["h"])] = res.get((r["w"], r["h"]), 0) + 1
278
+ common_res = max(res, key=res.get)
279
+ for r in ok:
280
+ why = []
281
+ if abs(r["bytes"] - mu) > 4 * sd:
282
+ why.append("size")
283
+ if res.get((r["w"], r["h"]), 0) < max(3, len(ok) * 0.001) \
284
+ and (r["w"], r["h"]) != common_res:
285
+ why.append(f"resolution {r['w']}x{r['h']}")
286
+ if r["bright"] is not None and (r["bright"] < 8 or r["bright"] > 247):
287
+ why.append("near-blank")
288
+ if why:
289
+ outlier_files.append(f"{r['path']} ({', '.join(why)})")
290
+
291
+ # labels
292
+ counts, file_map, label_sources = _read_labels(root)
293
+ imbalance = None
294
+ if counts and len(counts) > 1:
295
+ total = sum(counts.values())
296
+ shares = sorted(((v / total, k) for k, v in counts.items()), reverse=True)
297
+ if shares[0][0] / max(shares[-1][0], 1e-9) > 3:
298
+ imbalance = shares
299
+ unlabeled, orphan_labels = [], []
300
+ if file_map:
301
+ img_names = {Path(r["path"]).name for r in ok}
302
+ img_stems = {Path(r["path"]).stem for r in ok}
303
+ keyset = set()
304
+ for k in file_map:
305
+ keyset.add(Path(k).name)
306
+ keyset.add(Path(k).stem)
307
+ unlabeled = sorted(n for n in img_names
308
+ if n not in keyset and Path(n).stem not in keyset)
309
+ orphan_labels = sorted(k for k in file_map
310
+ if Path(k).name not in img_names
311
+ and Path(k).stem not in img_stems)
312
+
313
+ # ---------------- score + findings
314
+ n = max(len(ok), 1)
315
+ findings, score = [], 100
316
+
317
+ def add(cond, sev, penalty, text, detail=""):
318
+ nonlocal score
319
+ if cond:
320
+ findings.append((sev, text, detail))
321
+ score -= penalty
322
+ return bool(cond)
323
+
324
+ findings.append(("ok", f"{len(ok):,} of {len(images):,} images readable", ""))
325
+ add(corrupted, "bad", min(20, int(len(corrupted) / n * 400) + 2),
326
+ f"{len(corrupted):,} corrupted images ({len(corrupted) / n:.1%})")
327
+ add(dup_files, "warn", min(15, int(len(dup_files) / n * 200) + 2),
328
+ f"{len(dup_files):,} exact duplicates in {len(dup_groups):,} groups "
329
+ f"({len(dup_files) / n:.1%})")
330
+ add(n_near, "warn", min(10, int(n_near / n * 100) + 1),
331
+ f"{n_near:,} near-duplicate images in {len(near_groups_idx):,} groups")
332
+ add(imbalance, "warn", 10 if imbalance and imbalance[0][0] > 0.6 else 5,
333
+ "Class imbalance detected" if imbalance else "",
334
+ " / ".join(f"{k} {s:.0%}" for s, k in imbalance[:6]) if imbalance else "")
335
+ add(leak_pairs, "bad", 15,
336
+ f"Possible train/test leakage — {len(leak_pairs):,} suspicious pairs")
337
+ add(outlier_files, "warn", min(5, int(len(outlier_files) / n * 100) + 1),
338
+ f"{len(outlier_files):,} anomalous images (size / resolution / near-blank)")
339
+ add(unlabeled, "warn", min(8, int(len(unlabeled) / n * 100) + 1),
340
+ f"{len(unlabeled):,} images have no row in any label file")
341
+ add(orphan_labels, "warn", 3,
342
+ f"{len(orphan_labels):,} label rows point to files that don't exist")
343
+ if not counts:
344
+ findings.append(("warn", "No parseable label file found "
345
+ "(need a CSV/TSV with a label/class column)", ""))
346
+ score = max(0, min(100, score))
347
+
348
+ # ---------------- terminal output
349
+ print()
350
+ grade = "g" if score >= 85 else "y" if score >= 60 else "r"
351
+ print("Dataset health " + _col(f"{score} / 100", grade, tty))
352
+ print()
353
+ for sev, text, detail in findings:
354
+ mark = {"ok": _col("✓", "g", tty), "warn": _col("⚠", "y", tty),
355
+ "bad": _col("✕", "r", tty)}[sev]
356
+ print(f" {mark} {text}")
357
+ if detail:
358
+ print(_col(f" {detail}", "d", tty))
359
+
360
+ # ---------------- fix mode
361
+ if fix and (dup_files or corrupted):
362
+ q = root / "_dset_quarantine"
363
+ moved = 0
364
+ for p in dup_files + corrupted:
365
+ src = Path(p)
366
+ dst = q / src.relative_to(root)
367
+ dst.parent.mkdir(parents=True, exist_ok=True)
368
+ try:
369
+ shutil.move(str(src), str(dst))
370
+ moved += 1
371
+ except OSError:
372
+ pass
373
+ print(f"\n Moved {moved:,} files (exact duplicates + corrupted) to "
374
+ f"{q.name}/ — review, then delete or restore them.")
375
+ elif fix:
376
+ print("\n Nothing to quarantine.")
377
+
378
+ # ---------------- html report
379
+ report = report_path or (root / "dset_report.html")
380
+ _write_html(report, {
381
+ "root": str(root), "score": score, "n_images": len(images),
382
+ "n_files": len(paths), "findings": findings,
383
+ "corrupted": corrupted, "dup_files": dup_files,
384
+ "near_files": near_files, "leak_pairs": leak_pairs,
385
+ "outlier_files": outlier_files, "unlabeled": unlabeled,
386
+ "orphan_labels": orphan_labels,
387
+ })
388
+ print(f"\n Report: {report} ({time.time() - t0:.1f}s)")
@@ -591,8 +591,21 @@ def main(argv=None):
591
591
  sp.add_argument("--force", action="store_true",
592
592
  help="discard uncommitted changes")
593
593
 
594
+ sp = sub.add_parser("check", help="scan a folder for dataset quality "
595
+ "problems (duplicates, corruption, leakage, ...)")
596
+ sp.add_argument("path", nargs="?", default=".",
597
+ help="folder to scan (default: current directory)")
598
+ sp.add_argument("--fix", action="store_true",
599
+ help="move exact duplicates and corrupted files "
600
+ "to _dset_quarantine/")
601
+ sp.add_argument("--report", help="path for the HTML report")
602
+
594
603
  args = p.parse_args(argv)
595
604
  try:
605
+ if args.cmd == "check":
606
+ from .check import run_check
607
+ run_check(args.path, fix=args.fix, report_path=args.report)
608
+ return
596
609
  {"init": cmd_init, "add": cmd_add, "status": cmd_status,
597
610
  "commit": cmd_commit, "log": cmd_log, "diff": cmd_diff,
598
611
  "checkout": cmd_checkout}[args.cmd](args)
@@ -1,9 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dset-cli
3
- Version: 0.1.0
3
+ Version: 0.2.1
4
4
  Summary: Version control for datasets — commit, diff, and roll back millions of files with semantic diffs.
5
5
  Requires-Python: >=3.9
6
6
  Description-Content-Type: text/markdown
7
+ Requires-Dist: Pillow>=9.0
7
8
 
8
9
  # dset — version control for datasets
9
10
 
@@ -69,6 +70,7 @@ dset checkout v1
69
70
  | `dset log` | list versions with file counts and sizes |
70
71
  | `dset diff <a> <b>` | semantic diff: file counts by type, size, class distribution shift |
71
72
  | `dset checkout <ref>` | restore the working tree to a version (`--force` to discard changes) |
73
+ | `dset check [path]` | quality scan: duplicates, corruption, leakage, imbalance, outliers (`--fix` to quarantine) |
72
74
 
73
75
  Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
74
76
 
@@ -88,6 +90,28 @@ Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
88
90
  corrupt history. On btrfs/XFS the copy is a free copy-on-write clone.
89
91
  - Uncommitted changes block `dset checkout` unless you pass `--force`.
90
92
 
93
+ ## dset check — the quality engine
94
+
95
+ ```
96
+ dset check ./my-dataset # scan and score
97
+ dset check ./my-dataset --fix # also quarantine exact dups + corrupted
98
+ ```
99
+
100
+ Scans every image (parallel, ~2,000 images/sec) and reports:
101
+
102
+ - corrupted / unreadable images
103
+ - exact duplicates (content hash) and near-duplicates (perceptual dhash + LSH)
104
+ - train/test leakage — exact and near-duplicate pairs across train/ and test/
105
+ - class imbalance (parsed from CSV/TSV label files)
106
+ - outliers: anomalous file size, odd resolutions, near-blank images
107
+ - images missing from label files, and label rows pointing to missing files
108
+
109
+ Prints a health score out of 100 and writes a browsable `dset_report.html`.
110
+ `--fix` never deletes anything — it moves exact duplicates and corrupted
111
+ files to `_dset_quarantine/` for you to review.
112
+
113
+ Requires Pillow (installed automatically with the package).
114
+
91
115
  ## Current limits (MVP)
92
116
 
93
117
  - Class stats read CSV/TSV label files only (COCO/YOLO parsers are next).
@@ -1,9 +1,11 @@
1
1
  README.md
2
2
  pyproject.toml
3
3
  dset/__init__.py
4
+ dset/check.py
4
5
  dset/cli.py
5
6
  dset_cli.egg-info/PKG-INFO
6
7
  dset_cli.egg-info/SOURCES.txt
7
8
  dset_cli.egg-info/dependency_links.txt
8
9
  dset_cli.egg-info/entry_points.txt
10
+ dset_cli.egg-info/requires.txt
9
11
  dset_cli.egg-info/top_level.txt
@@ -0,0 +1 @@
1
+ Pillow>=9.0
@@ -1,8 +1,9 @@
1
1
  [project]
2
2
  name = "dset-cli"
3
- version = "0.1.0"
3
+ version = "0.2.1"
4
4
  description = "Version control for datasets — commit, diff, and roll back millions of files with semantic diffs."
5
5
  requires-python = ">=3.9"
6
+ dependencies = ["Pillow>=9.0"]
6
7
  readme = "README.md"
7
8
 
8
9
  [project.scripts]
File without changes
File without changes