dset-cli 0.1.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dset_cli-0.1.0 → dset_cli-0.2.1}/PKG-INFO +25 -1
- {dset_cli-0.1.0 → dset_cli-0.2.1}/README.md +23 -0
- dset_cli-0.2.1/dset/check.py +388 -0
- {dset_cli-0.1.0 → dset_cli-0.2.1}/dset/cli.py +13 -0
- {dset_cli-0.1.0 → dset_cli-0.2.1}/dset_cli.egg-info/PKG-INFO +25 -1
- {dset_cli-0.1.0 → dset_cli-0.2.1}/dset_cli.egg-info/SOURCES.txt +2 -0
- dset_cli-0.2.1/dset_cli.egg-info/requires.txt +1 -0
- {dset_cli-0.1.0 → dset_cli-0.2.1}/pyproject.toml +2 -1
- {dset_cli-0.1.0 → dset_cli-0.2.1}/dset/__init__.py +0 -0
- {dset_cli-0.1.0 → dset_cli-0.2.1}/dset_cli.egg-info/dependency_links.txt +0 -0
- {dset_cli-0.1.0 → dset_cli-0.2.1}/dset_cli.egg-info/entry_points.txt +0 -0
- {dset_cli-0.1.0 → dset_cli-0.2.1}/dset_cli.egg-info/top_level.txt +0 -0
- {dset_cli-0.1.0 → dset_cli-0.2.1}/setup.cfg +0 -0
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dset-cli
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Version control for datasets — commit, diff, and roll back millions of files with semantic diffs.
|
|
5
5
|
Requires-Python: >=3.9
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: Pillow>=9.0
|
|
7
8
|
|
|
8
9
|
# dset — version control for datasets
|
|
9
10
|
|
|
@@ -69,6 +70,7 @@ dset checkout v1
|
|
|
69
70
|
| `dset log` | list versions with file counts and sizes |
|
|
70
71
|
| `dset diff <a> <b>` | semantic diff: file counts by type, size, class distribution shift |
|
|
71
72
|
| `dset checkout <ref>` | restore the working tree to a version (`--force` to discard changes) |
|
|
73
|
+
| `dset check [path]` | quality scan: duplicates, corruption, leakage, imbalance, outliers (`--fix` to quarantine) |
|
|
72
74
|
|
|
73
75
|
Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
|
|
74
76
|
|
|
@@ -88,6 +90,28 @@ Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
|
|
|
88
90
|
corrupt history. On btrfs/XFS the copy is a free copy-on-write clone.
|
|
89
91
|
- Uncommitted changes block `dset checkout` unless you pass `--force`.
|
|
90
92
|
|
|
93
|
+
## dset check — the quality engine
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
dset check ./my-dataset # scan and score
|
|
97
|
+
dset check ./my-dataset --fix # also quarantine exact dups + corrupted
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Scans every image (parallel, ~2,000 images/sec) and reports:
|
|
101
|
+
|
|
102
|
+
- corrupted / unreadable images
|
|
103
|
+
- exact duplicates (content hash) and near-duplicates (perceptual dhash + LSH)
|
|
104
|
+
- train/test leakage — exact and near-duplicate pairs across train/ and test/
|
|
105
|
+
- class imbalance (parsed from CSV/TSV label files)
|
|
106
|
+
- outliers: anomalous file size, odd resolutions, near-blank images
|
|
107
|
+
- images missing from label files, and label rows pointing to missing files
|
|
108
|
+
|
|
109
|
+
Prints a health score out of 100 and writes a browsable `dset_report.html`.
|
|
110
|
+
`--fix` never deletes anything — it moves exact duplicates and corrupted
|
|
111
|
+
files to `_dset_quarantine/` for you to review.
|
|
112
|
+
|
|
113
|
+
Requires Pillow (installed automatically with the package).
|
|
114
|
+
|
|
91
115
|
## Current limits (MVP)
|
|
92
116
|
|
|
93
117
|
- Class stats read CSV/TSV label files only (COCO/YOLO parsers are next).
|
|
@@ -62,6 +62,7 @@ dset checkout v1
|
|
|
62
62
|
| `dset log` | list versions with file counts and sizes |
|
|
63
63
|
| `dset diff <a> <b>` | semantic diff: file counts by type, size, class distribution shift |
|
|
64
64
|
| `dset checkout <ref>` | restore the working tree to a version (`--force` to discard changes) |
|
|
65
|
+
| `dset check [path]` | quality scan: duplicates, corruption, leakage, imbalance, outliers (`--fix` to quarantine) |
|
|
65
66
|
|
|
66
67
|
Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
|
|
67
68
|
|
|
@@ -81,6 +82,28 @@ Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
|
|
|
81
82
|
corrupt history. On btrfs/XFS the copy is a free copy-on-write clone.
|
|
82
83
|
- Uncommitted changes block `dset checkout` unless you pass `--force`.
|
|
83
84
|
|
|
85
|
+
## dset check — the quality engine
|
|
86
|
+
|
|
87
|
+
```
|
|
88
|
+
dset check ./my-dataset # scan and score
|
|
89
|
+
dset check ./my-dataset --fix # also quarantine exact dups + corrupted
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Scans every image (parallel, ~2,000 images/sec) and reports:
|
|
93
|
+
|
|
94
|
+
- corrupted / unreadable images
|
|
95
|
+
- exact duplicates (content hash) and near-duplicates (perceptual dhash + LSH)
|
|
96
|
+
- train/test leakage — exact and near-duplicate pairs across train/ and test/
|
|
97
|
+
- class imbalance (parsed from CSV/TSV label files)
|
|
98
|
+
- outliers: anomalous file size, odd resolutions, near-blank images
|
|
99
|
+
- images missing from label files, and label rows pointing to missing files
|
|
100
|
+
|
|
101
|
+
Prints a health score out of 100 and writes a browsable `dset_report.html`.
|
|
102
|
+
`--fix` never deletes anything — it moves exact duplicates and corrupted
|
|
103
|
+
files to `_dset_quarantine/` for you to review.
|
|
104
|
+
|
|
105
|
+
Requires Pillow (installed automatically with the package).
|
|
106
|
+
|
|
84
107
|
## Current limits (MVP)
|
|
85
108
|
|
|
86
109
|
- Class stats read CSV/TSV label files only (COCO/YOLO parsers are next).
|
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
"""dset check — the dataset quality engine.
|
|
2
|
+
|
|
3
|
+
Scans a folder of images + label files and reports:
|
|
4
|
+
corrupted images, exact duplicates, near-duplicates, class imbalance,
|
|
5
|
+
train/test leakage, outliers (size / resolution / brightness),
|
|
6
|
+
and label <-> file mismatches.
|
|
7
|
+
|
|
8
|
+
Writes a dset_report.html and prints a health score.
|
|
9
|
+
`--fix` moves exact duplicates and corrupted files into _dset_quarantine/.
|
|
10
|
+
|
|
11
|
+
Requires Pillow (pip install Pillow). Everything else is stdlib.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import csv
|
|
15
|
+
import hashlib
|
|
16
|
+
import html
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import shutil
|
|
20
|
+
import statistics
|
|
21
|
+
import sys
|
|
22
|
+
import time
|
|
23
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
IMAGE_EXT = {".jpg", ".jpeg", ".png", ".tif", ".tiff", ".bmp", ".gif", ".webp"}
|
|
27
|
+
CLASS_COLUMNS = ("label", "class", "class_name", "category", "label_name", "target")
|
|
28
|
+
FILE_COLUMNS = ("file", "filename", "path", "image", "id", "name")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _col(s, c, tty):
|
|
32
|
+
codes = {"g": "32", "r": "31", "y": "33", "d": "90", "b": "1"}
|
|
33
|
+
return f"\033[{codes[c]}m{s}\033[0m" if tty else str(s)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ---------------------------------------------------------------- per-image
|
|
37
|
+
|
|
38
|
+
def _analyze_image(path):
|
|
39
|
+
"""Hash + decode one image. Returns a record dict; ok=False if corrupt."""
|
|
40
|
+
from PIL import Image
|
|
41
|
+
rec = {"path": str(path), "bytes": 0, "sha": None, "ok": False,
|
|
42
|
+
"w": 0, "h": 0, "dhash": None, "bright": None}
|
|
43
|
+
try:
|
|
44
|
+
data = path.read_bytes()
|
|
45
|
+
except OSError:
|
|
46
|
+
return rec
|
|
47
|
+
rec["bytes"] = len(data)
|
|
48
|
+
rec["sha"] = hashlib.sha256(data).hexdigest()
|
|
49
|
+
try:
|
|
50
|
+
with Image.open(path) as im:
|
|
51
|
+
im.load()
|
|
52
|
+
rec["w"], rec["h"] = im.size
|
|
53
|
+
g = im.convert("L").resize((9, 8))
|
|
54
|
+
px = list(g.getdata())
|
|
55
|
+
bits = 0
|
|
56
|
+
for r in range(8):
|
|
57
|
+
for c in range(8):
|
|
58
|
+
bits = (bits << 1) | (px[r * 9 + c] > px[r * 9 + c + 1])
|
|
59
|
+
rec["dhash"] = bits
|
|
60
|
+
rec["bright"] = sum(px) / len(px)
|
|
61
|
+
rec["ok"] = True
|
|
62
|
+
except Exception:
|
|
63
|
+
pass
|
|
64
|
+
return rec
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _hamming(a, b):
|
|
68
|
+
return bin(a ^ b).count("1")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _near_dup_groups(recs, max_dist=6):
|
|
72
|
+
"""LSH banding on the 64-bit dhash: candidates share one 16-bit band."""
|
|
73
|
+
buckets = {}
|
|
74
|
+
for i, r in enumerate(recs):
|
|
75
|
+
hsh = r["dhash"]
|
|
76
|
+
for band in range(4):
|
|
77
|
+
key = (band, (hsh >> (band * 16)) & 0xFFFF)
|
|
78
|
+
buckets.setdefault(key, []).append(i)
|
|
79
|
+
parent = list(range(len(recs)))
|
|
80
|
+
|
|
81
|
+
def find(x):
|
|
82
|
+
while parent[x] != x:
|
|
83
|
+
parent[x] = parent[parent[x]]
|
|
84
|
+
x = parent[x]
|
|
85
|
+
return x
|
|
86
|
+
|
|
87
|
+
checked = set()
|
|
88
|
+
for idxs in buckets.values():
|
|
89
|
+
if len(idxs) < 2 or len(idxs) > 400: # skip degenerate mega-buckets
|
|
90
|
+
continue
|
|
91
|
+
for i in range(len(idxs)):
|
|
92
|
+
for j in range(i + 1, len(idxs)):
|
|
93
|
+
a, b = idxs[i], idxs[j]
|
|
94
|
+
if (a, b) in checked:
|
|
95
|
+
continue
|
|
96
|
+
checked.add((a, b))
|
|
97
|
+
if _hamming(recs[a]["dhash"], recs[b]["dhash"]) <= max_dist:
|
|
98
|
+
parent[find(a)] = find(b)
|
|
99
|
+
groups = {}
|
|
100
|
+
for i in range(len(recs)):
|
|
101
|
+
groups.setdefault(find(i), []).append(i)
|
|
102
|
+
return [g for g in groups.values() if len(g) > 1]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# ---------------------------------------------------------------- labels
|
|
106
|
+
|
|
107
|
+
def _read_labels(root):
|
|
108
|
+
"""Parse CSV/TSV label files. Returns (class_counts, file->class, sources)."""
|
|
109
|
+
counts, file_map, sources = {}, {}, []
|
|
110
|
+
for p in sorted(root.rglob("*")):
|
|
111
|
+
if p.suffix.lower() not in (".csv", ".tsv") or not p.is_file():
|
|
112
|
+
continue
|
|
113
|
+
if any(part.startswith((".", "_dset")) for part in p.parts):
|
|
114
|
+
continue
|
|
115
|
+
delim = "\t" if p.suffix.lower() == ".tsv" else ","
|
|
116
|
+
try:
|
|
117
|
+
with open(p, newline="", encoding="utf-8", errors="replace") as f:
|
|
118
|
+
reader = csv.reader(f, delimiter=delim)
|
|
119
|
+
header = next(reader, None)
|
|
120
|
+
if not header:
|
|
121
|
+
continue
|
|
122
|
+
lower = [h.strip().lower() for h in header]
|
|
123
|
+
ci = next((lower.index(c) for c in CLASS_COLUMNS if c in lower), None)
|
|
124
|
+
fi = next((lower.index(c) for c in FILE_COLUMNS if c in lower), None)
|
|
125
|
+
if ci is None:
|
|
126
|
+
continue
|
|
127
|
+
sources.append(p)
|
|
128
|
+
for row in reader:
|
|
129
|
+
if ci < len(row) and row[ci].strip():
|
|
130
|
+
v = row[ci].strip()
|
|
131
|
+
counts[v] = counts.get(v, 0) + 1
|
|
132
|
+
if fi is not None and fi < len(row):
|
|
133
|
+
file_map[row[fi].strip()] = v
|
|
134
|
+
except OSError:
|
|
135
|
+
continue
|
|
136
|
+
return counts, file_map, sources
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# ---------------------------------------------------------------- report
|
|
140
|
+
|
|
141
|
+
def _write_html(out, ctx):
|
|
142
|
+
rows = []
|
|
143
|
+
for sev, text, detail in ctx["findings"]:
|
|
144
|
+
color = {"ok": "#0a7d33", "warn": "#b0771e", "bad": "#c0392b"}[sev]
|
|
145
|
+
mark = {"ok": "✓", "warn": "⚠", "bad": "✕"}[sev]
|
|
146
|
+
d = f'<div class="detail">{html.escape(detail)}</div>' if detail else ""
|
|
147
|
+
rows.append(f'<div class="finding"><span style="color:{color}">{mark}</span>'
|
|
148
|
+
f'<div>{html.escape(text)}{d}</div></div>')
|
|
149
|
+
|
|
150
|
+
def listing(title, items):
|
|
151
|
+
if not items:
|
|
152
|
+
return ""
|
|
153
|
+
lis = "".join(f"<li>{html.escape(i)}</li>" for i in items[:60])
|
|
154
|
+
more = (f'<p class="more">… and {len(items) - 60:,} more</p>'
|
|
155
|
+
if len(items) > 60 else "")
|
|
156
|
+
return (f'<details><summary>{html.escape(title)} ({len(items):,})</summary>'
|
|
157
|
+
f"<ul>{lis}</ul>{more}</details>")
|
|
158
|
+
|
|
159
|
+
score = ctx["score"]
|
|
160
|
+
color = "#0a7d33" if score >= 85 else "#b0771e" if score >= 60 else "#c0392b"
|
|
161
|
+
doc = f"""<!DOCTYPE html><html><head><meta charset="utf-8">
|
|
162
|
+
<title>dset report — {html.escape(ctx['root'])}</title><style>
|
|
163
|
+
body{{font-family:system-ui,sans-serif;max-width:760px;margin:40px auto;
|
|
164
|
+
padding:0 24px;color:#111;line-height:1.6}}
|
|
165
|
+
h1{{font-size:20px;font-weight:600}} .score{{font-size:52px;font-weight:600;
|
|
166
|
+
color:{color};margin:8px 0 4px}} .meta{{color:#777;font-size:14px}}
|
|
167
|
+
.finding{{display:flex;gap:12px;padding:10px 0;border-bottom:1px solid #eee;
|
|
168
|
+
font-size:15px}} .detail{{color:#777;font-size:13px}}
|
|
169
|
+
details{{margin:14px 0;font-size:14px}} summary{{cursor:pointer;font-weight:500}}
|
|
170
|
+
ul{{color:#555;font-family:ui-monospace,monospace;font-size:12.5px}}
|
|
171
|
+
.more{{color:#999;font-size:13px}}</style></head><body>
|
|
172
|
+
<h1>dset check — {html.escape(ctx['root'])}</h1>
|
|
173
|
+
<div class="score">{score} / 100</div>
|
|
174
|
+
<div class="meta">{ctx['n_images']:,} images · {ctx['n_files']:,} files scanned ·
|
|
175
|
+
{time.strftime('%Y-%m-%d %H:%M')}</div><br>
|
|
176
|
+
{''.join(rows)}
|
|
177
|
+
{listing('Corrupted files', ctx['corrupted'])}
|
|
178
|
+
{listing('Exact duplicate files (kept one per group)', ctx['dup_files'])}
|
|
179
|
+
{listing('Near-duplicate groups (first file of each)', ctx['near_files'])}
|
|
180
|
+
{listing('Suspected train/test leakage pairs', ctx['leak_pairs'])}
|
|
181
|
+
{listing('Outlier images', ctx['outlier_files'])}
|
|
182
|
+
{listing('Images missing from label files', ctx['unlabeled'])}
|
|
183
|
+
{listing('Label rows pointing to missing files', ctx['orphan_labels'])}
|
|
184
|
+
</body></html>"""
|
|
185
|
+
Path(out).write_text(doc)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
# ---------------------------------------------------------------- command
|
|
189
|
+
|
|
190
|
+
def run_check(root_arg, fix=False, report_path=None):
|
|
191
|
+
try:
|
|
192
|
+
import PIL # noqa: F401
|
|
193
|
+
except ImportError:
|
|
194
|
+
print("dset check needs Pillow: pip install Pillow", file=sys.stderr)
|
|
195
|
+
sys.exit(1)
|
|
196
|
+
|
|
197
|
+
tty = sys.stdout.isatty() and os.environ.get("NO_COLOR") is None
|
|
198
|
+
root = Path(root_arg).resolve()
|
|
199
|
+
if not root.exists():
|
|
200
|
+
print(f"error: path not found: {root}", file=sys.stderr)
|
|
201
|
+
sys.exit(1)
|
|
202
|
+
|
|
203
|
+
paths = [p for p in sorted(root.rglob("*"))
|
|
204
|
+
if p.is_file()
|
|
205
|
+
and not any(part.startswith((".", "_dset")) for part in p.parts)]
|
|
206
|
+
images = [p for p in paths if p.suffix.lower() in IMAGE_EXT]
|
|
207
|
+
print(f"Analyzing {len(images):,} images ({len(paths):,} files) in {root} ...")
|
|
208
|
+
|
|
209
|
+
t0 = time.time()
|
|
210
|
+
recs = []
|
|
211
|
+
with ThreadPoolExecutor(max_workers=min(32, (os.cpu_count() or 4) * 2)) as ex:
|
|
212
|
+
for i, rec in enumerate(ex.map(_analyze_image, images), 1):
|
|
213
|
+
recs.append(rec)
|
|
214
|
+
if i % 2000 == 0:
|
|
215
|
+
print(f"\r {i:,}/{len(images):,} ...", end="", flush=True)
|
|
216
|
+
print("\r" + " " * 30 + "\r", end="")
|
|
217
|
+
|
|
218
|
+
ok = [r for r in recs if r["ok"]]
|
|
219
|
+
corrupted = [r["path"] for r in recs if not r["ok"]]
|
|
220
|
+
|
|
221
|
+
# exact duplicates (by content hash)
|
|
222
|
+
by_sha = {}
|
|
223
|
+
for r in ok:
|
|
224
|
+
by_sha.setdefault(r["sha"], []).append(r["path"])
|
|
225
|
+
dup_groups = [v for v in by_sha.values() if len(v) > 1]
|
|
226
|
+
dup_files = [p for g in dup_groups for p in g[1:]] # all but first
|
|
227
|
+
|
|
228
|
+
# near duplicates (exclude exact dups so groups aren't double-counted)
|
|
229
|
+
firsts = {g[0] for g in dup_groups}
|
|
230
|
+
uniq = [r for r in ok if r["sha"] in {k for k, v in by_sha.items()}
|
|
231
|
+
and (len(by_sha[r["sha"]]) == 1 or r["path"] in firsts)]
|
|
232
|
+
near_groups_idx = _near_dup_groups(uniq)
|
|
233
|
+
near_files = [uniq[g[0]]["path"] for g in near_groups_idx]
|
|
234
|
+
n_near = sum(len(g) for g in near_groups_idx) - len(near_groups_idx)
|
|
235
|
+
|
|
236
|
+
# train/test leakage
|
|
237
|
+
def split_of(p):
|
|
238
|
+
parts = [x.lower() for x in Path(p).parts]
|
|
239
|
+
if any("train" in x for x in parts):
|
|
240
|
+
return "train"
|
|
241
|
+
if any(("test" in x or "val" in x) for x in parts):
|
|
242
|
+
return "test"
|
|
243
|
+
return None
|
|
244
|
+
|
|
245
|
+
leak_pairs = []
|
|
246
|
+
tr = {r["sha"]: r["path"] for r in ok if split_of(r["path"]) == "train"}
|
|
247
|
+
te = [r for r in ok if split_of(r["path"]) == "test"]
|
|
248
|
+
for r in te:
|
|
249
|
+
if r["sha"] in tr:
|
|
250
|
+
leak_pairs.append(f"{tr[r['sha']]} == {r['path']}")
|
|
251
|
+
if te and tr:
|
|
252
|
+
tr_hashes = {}
|
|
253
|
+
for r in ok:
|
|
254
|
+
if split_of(r["path"]) == "train":
|
|
255
|
+
for band in range(4):
|
|
256
|
+
key = (band, (r["dhash"] >> (band * 16)) & 0xFFFF)
|
|
257
|
+
tr_hashes.setdefault(key, []).append(r)
|
|
258
|
+
seen = set()
|
|
259
|
+
for r in te:
|
|
260
|
+
for band in range(4):
|
|
261
|
+
key = (band, (r["dhash"] >> (band * 16)) & 0xFFFF)
|
|
262
|
+
for cand in tr_hashes.get(key, [])[:200]:
|
|
263
|
+
pair = (cand["path"], r["path"])
|
|
264
|
+
if pair in seen or cand["sha"] == r["sha"]:
|
|
265
|
+
continue
|
|
266
|
+
seen.add(pair)
|
|
267
|
+
if _hamming(cand["dhash"], r["dhash"]) <= 6:
|
|
268
|
+
leak_pairs.append(f"{cand['path']} ≈ {r['path']}")
|
|
269
|
+
|
|
270
|
+
# outliers: file size, resolution, brightness
|
|
271
|
+
outlier_files = []
|
|
272
|
+
if len(ok) >= 20:
|
|
273
|
+
sizes = [r["bytes"] for r in ok]
|
|
274
|
+
mu, sd = statistics.mean(sizes), statistics.pstdev(sizes) or 1
|
|
275
|
+
res = {}
|
|
276
|
+
for r in ok:
|
|
277
|
+
res[(r["w"], r["h"])] = res.get((r["w"], r["h"]), 0) + 1
|
|
278
|
+
common_res = max(res, key=res.get)
|
|
279
|
+
for r in ok:
|
|
280
|
+
why = []
|
|
281
|
+
if abs(r["bytes"] - mu) > 4 * sd:
|
|
282
|
+
why.append("size")
|
|
283
|
+
if res.get((r["w"], r["h"]), 0) < max(3, len(ok) * 0.001) \
|
|
284
|
+
and (r["w"], r["h"]) != common_res:
|
|
285
|
+
why.append(f"resolution {r['w']}x{r['h']}")
|
|
286
|
+
if r["bright"] is not None and (r["bright"] < 8 or r["bright"] > 247):
|
|
287
|
+
why.append("near-blank")
|
|
288
|
+
if why:
|
|
289
|
+
outlier_files.append(f"{r['path']} ({', '.join(why)})")
|
|
290
|
+
|
|
291
|
+
# labels
|
|
292
|
+
counts, file_map, label_sources = _read_labels(root)
|
|
293
|
+
imbalance = None
|
|
294
|
+
if counts and len(counts) > 1:
|
|
295
|
+
total = sum(counts.values())
|
|
296
|
+
shares = sorted(((v / total, k) for k, v in counts.items()), reverse=True)
|
|
297
|
+
if shares[0][0] / max(shares[-1][0], 1e-9) > 3:
|
|
298
|
+
imbalance = shares
|
|
299
|
+
unlabeled, orphan_labels = [], []
|
|
300
|
+
if file_map:
|
|
301
|
+
img_names = {Path(r["path"]).name for r in ok}
|
|
302
|
+
img_stems = {Path(r["path"]).stem for r in ok}
|
|
303
|
+
keyset = set()
|
|
304
|
+
for k in file_map:
|
|
305
|
+
keyset.add(Path(k).name)
|
|
306
|
+
keyset.add(Path(k).stem)
|
|
307
|
+
unlabeled = sorted(n for n in img_names
|
|
308
|
+
if n not in keyset and Path(n).stem not in keyset)
|
|
309
|
+
orphan_labels = sorted(k for k in file_map
|
|
310
|
+
if Path(k).name not in img_names
|
|
311
|
+
and Path(k).stem not in img_stems)
|
|
312
|
+
|
|
313
|
+
# ---------------- score + findings
|
|
314
|
+
n = max(len(ok), 1)
|
|
315
|
+
findings, score = [], 100
|
|
316
|
+
|
|
317
|
+
def add(cond, sev, penalty, text, detail=""):
|
|
318
|
+
nonlocal score
|
|
319
|
+
if cond:
|
|
320
|
+
findings.append((sev, text, detail))
|
|
321
|
+
score -= penalty
|
|
322
|
+
return bool(cond)
|
|
323
|
+
|
|
324
|
+
findings.append(("ok", f"{len(ok):,} of {len(images):,} images readable", ""))
|
|
325
|
+
add(corrupted, "bad", min(20, int(len(corrupted) / n * 400) + 2),
|
|
326
|
+
f"{len(corrupted):,} corrupted images ({len(corrupted) / n:.1%})")
|
|
327
|
+
add(dup_files, "warn", min(15, int(len(dup_files) / n * 200) + 2),
|
|
328
|
+
f"{len(dup_files):,} exact duplicates in {len(dup_groups):,} groups "
|
|
329
|
+
f"({len(dup_files) / n:.1%})")
|
|
330
|
+
add(n_near, "warn", min(10, int(n_near / n * 100) + 1),
|
|
331
|
+
f"{n_near:,} near-duplicate images in {len(near_groups_idx):,} groups")
|
|
332
|
+
add(imbalance, "warn", 10 if imbalance and imbalance[0][0] > 0.6 else 5,
|
|
333
|
+
"Class imbalance detected" if imbalance else "",
|
|
334
|
+
" / ".join(f"{k} {s:.0%}" for s, k in imbalance[:6]) if imbalance else "")
|
|
335
|
+
add(leak_pairs, "bad", 15,
|
|
336
|
+
f"Possible train/test leakage — {len(leak_pairs):,} suspicious pairs")
|
|
337
|
+
add(outlier_files, "warn", min(5, int(len(outlier_files) / n * 100) + 1),
|
|
338
|
+
f"{len(outlier_files):,} anomalous images (size / resolution / near-blank)")
|
|
339
|
+
add(unlabeled, "warn", min(8, int(len(unlabeled) / n * 100) + 1),
|
|
340
|
+
f"{len(unlabeled):,} images have no row in any label file")
|
|
341
|
+
add(orphan_labels, "warn", 3,
|
|
342
|
+
f"{len(orphan_labels):,} label rows point to files that don't exist")
|
|
343
|
+
if not counts:
|
|
344
|
+
findings.append(("warn", "No parseable label file found "
|
|
345
|
+
"(need a CSV/TSV with a label/class column)", ""))
|
|
346
|
+
score = max(0, min(100, score))
|
|
347
|
+
|
|
348
|
+
# ---------------- terminal output
|
|
349
|
+
print()
|
|
350
|
+
grade = "g" if score >= 85 else "y" if score >= 60 else "r"
|
|
351
|
+
print("Dataset health " + _col(f"{score} / 100", grade, tty))
|
|
352
|
+
print()
|
|
353
|
+
for sev, text, detail in findings:
|
|
354
|
+
mark = {"ok": _col("✓", "g", tty), "warn": _col("⚠", "y", tty),
|
|
355
|
+
"bad": _col("✕", "r", tty)}[sev]
|
|
356
|
+
print(f" {mark} {text}")
|
|
357
|
+
if detail:
|
|
358
|
+
print(_col(f" {detail}", "d", tty))
|
|
359
|
+
|
|
360
|
+
# ---------------- fix mode
|
|
361
|
+
if fix and (dup_files or corrupted):
|
|
362
|
+
q = root / "_dset_quarantine"
|
|
363
|
+
moved = 0
|
|
364
|
+
for p in dup_files + corrupted:
|
|
365
|
+
src = Path(p)
|
|
366
|
+
dst = q / src.relative_to(root)
|
|
367
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
368
|
+
try:
|
|
369
|
+
shutil.move(str(src), str(dst))
|
|
370
|
+
moved += 1
|
|
371
|
+
except OSError:
|
|
372
|
+
pass
|
|
373
|
+
print(f"\n Moved {moved:,} files (exact duplicates + corrupted) to "
|
|
374
|
+
f"{q.name}/ — review, then delete or restore them.")
|
|
375
|
+
elif fix:
|
|
376
|
+
print("\n Nothing to quarantine.")
|
|
377
|
+
|
|
378
|
+
# ---------------- html report
|
|
379
|
+
report = report_path or (root / "dset_report.html")
|
|
380
|
+
_write_html(report, {
|
|
381
|
+
"root": str(root), "score": score, "n_images": len(images),
|
|
382
|
+
"n_files": len(paths), "findings": findings,
|
|
383
|
+
"corrupted": corrupted, "dup_files": dup_files,
|
|
384
|
+
"near_files": near_files, "leak_pairs": leak_pairs,
|
|
385
|
+
"outlier_files": outlier_files, "unlabeled": unlabeled,
|
|
386
|
+
"orphan_labels": orphan_labels,
|
|
387
|
+
})
|
|
388
|
+
print(f"\n Report: {report} ({time.time() - t0:.1f}s)")
|
|
@@ -591,8 +591,21 @@ def main(argv=None):
|
|
|
591
591
|
sp.add_argument("--force", action="store_true",
|
|
592
592
|
help="discard uncommitted changes")
|
|
593
593
|
|
|
594
|
+
sp = sub.add_parser("check", help="scan a folder for dataset quality "
|
|
595
|
+
"problems (duplicates, corruption, leakage, ...)")
|
|
596
|
+
sp.add_argument("path", nargs="?", default=".",
|
|
597
|
+
help="folder to scan (default: current directory)")
|
|
598
|
+
sp.add_argument("--fix", action="store_true",
|
|
599
|
+
help="move exact duplicates and corrupted files "
|
|
600
|
+
"to _dset_quarantine/")
|
|
601
|
+
sp.add_argument("--report", help="path for the HTML report")
|
|
602
|
+
|
|
594
603
|
args = p.parse_args(argv)
|
|
595
604
|
try:
|
|
605
|
+
if args.cmd == "check":
|
|
606
|
+
from .check import run_check
|
|
607
|
+
run_check(args.path, fix=args.fix, report_path=args.report)
|
|
608
|
+
return
|
|
596
609
|
{"init": cmd_init, "add": cmd_add, "status": cmd_status,
|
|
597
610
|
"commit": cmd_commit, "log": cmd_log, "diff": cmd_diff,
|
|
598
611
|
"checkout": cmd_checkout}[args.cmd](args)
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dset-cli
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Version control for datasets — commit, diff, and roll back millions of files with semantic diffs.
|
|
5
5
|
Requires-Python: >=3.9
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: Pillow>=9.0
|
|
7
8
|
|
|
8
9
|
# dset — version control for datasets
|
|
9
10
|
|
|
@@ -69,6 +70,7 @@ dset checkout v1
|
|
|
69
70
|
| `dset log` | list versions with file counts and sizes |
|
|
70
71
|
| `dset diff <a> <b>` | semantic diff: file counts by type, size, class distribution shift |
|
|
71
72
|
| `dset checkout <ref>` | restore the working tree to a version (`--force` to discard changes) |
|
|
73
|
+
| `dset check [path]` | quality scan: duplicates, corruption, leakage, imbalance, outliers (`--fix` to quarantine) |
|
|
72
74
|
|
|
73
75
|
Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
|
|
74
76
|
|
|
@@ -88,6 +90,28 @@ Refs can be a tag (`v3`), `HEAD`, or a commit-id prefix.
|
|
|
88
90
|
corrupt history. On btrfs/XFS the copy is a free copy-on-write clone.
|
|
89
91
|
- Uncommitted changes block `dset checkout` unless you pass `--force`.
|
|
90
92
|
|
|
93
|
+
## dset check — the quality engine
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
dset check ./my-dataset # scan and score
|
|
97
|
+
dset check ./my-dataset --fix # also quarantine exact dups + corrupted
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Scans every image (parallel, ~2,000 images/sec) and reports:
|
|
101
|
+
|
|
102
|
+
- corrupted / unreadable images
|
|
103
|
+
- exact duplicates (content hash) and near-duplicates (perceptual dhash + LSH)
|
|
104
|
+
- train/test leakage — exact and near-duplicate pairs across train/ and test/
|
|
105
|
+
- class imbalance (parsed from CSV/TSV label files)
|
|
106
|
+
- outliers: anomalous file size, odd resolutions, near-blank images
|
|
107
|
+
- images missing from label files, and label rows pointing to missing files
|
|
108
|
+
|
|
109
|
+
Prints a health score out of 100 and writes a browsable `dset_report.html`.
|
|
110
|
+
`--fix` never deletes anything — it moves exact duplicates and corrupted
|
|
111
|
+
files to `_dset_quarantine/` for you to review.
|
|
112
|
+
|
|
113
|
+
Requires Pillow (installed automatically with the package).
|
|
114
|
+
|
|
91
115
|
## Current limits (MVP)
|
|
92
116
|
|
|
93
117
|
- Class stats read CSV/TSV label files only (COCO/YOLO parsers are next).
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
README.md
|
|
2
2
|
pyproject.toml
|
|
3
3
|
dset/__init__.py
|
|
4
|
+
dset/check.py
|
|
4
5
|
dset/cli.py
|
|
5
6
|
dset_cli.egg-info/PKG-INFO
|
|
6
7
|
dset_cli.egg-info/SOURCES.txt
|
|
7
8
|
dset_cli.egg-info/dependency_links.txt
|
|
8
9
|
dset_cli.egg-info/entry_points.txt
|
|
10
|
+
dset_cli.egg-info/requires.txt
|
|
9
11
|
dset_cli.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Pillow>=9.0
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dset-cli"
|
|
3
|
-
version = "0.1
|
|
3
|
+
version = "0.2.1"
|
|
4
4
|
description = "Version control for datasets — commit, diff, and roll back millions of files with semantic diffs."
|
|
5
5
|
requires-python = ">=3.9"
|
|
6
|
+
dependencies = ["Pillow>=9.0"]
|
|
6
7
|
readme = "README.md"
|
|
7
8
|
|
|
8
9
|
[project.scripts]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|