labeltify 0.4.1__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {labeltify-0.4.1 → labeltify-0.4.2}/PKG-INFO +1 -1
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify/client.py +70 -4
- labeltify-0.4.2/labeltify/corrupt.py +189 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify.egg-info/PKG-INFO +1 -1
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify.egg-info/SOURCES.txt +2 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/pyproject.toml +1 -1
- {labeltify-0.4.1 → labeltify-0.4.2}/tests/test_client.py +39 -1
- labeltify-0.4.2/tests/test_corrupt.py +88 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/README.md +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify/__init__.py +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify/__main__.py +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify/dhash.py +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify/download.py +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify.egg-info/dependency_links.txt +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify.egg-info/entry_points.txt +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify.egg-info/requires.txt +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/labeltify.egg-info/top_level.txt +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/setup.cfg +0 -0
- {labeltify-0.4.1 → labeltify-0.4.2}/tests/test_download.py +0 -0
|
@@ -304,6 +304,8 @@ class LabelTifyClient:
|
|
|
304
304
|
camera: str | None = None,
|
|
305
305
|
pipeline: str | None = None,
|
|
306
306
|
trigger: str | None = None,
|
|
307
|
+
detect_corrupt: bool = True,
|
|
308
|
+
skip_corrupt: bool = False,
|
|
307
309
|
max_request_bytes: int = MAX_REQUEST_BYTES,
|
|
308
310
|
max_request_images: int = MAX_REQUEST_IMAGES,
|
|
309
311
|
) -> UploadReport:
|
|
@@ -312,6 +314,9 @@ class LabelTifyClient:
|
|
|
312
314
|
Every file is a ``file`` part; a file whose name ends in a label extension (.json, .jsonl, .xml,
|
|
313
315
|
.txt, .yaml, .yml, .names, .csv, .zip) is a label file, anything else an image. With any label
|
|
314
316
|
file, ``work_state`` is required. ``dhashes`` (16 hex characters each) go one per image, in order.
|
|
317
|
+
With Pillow installed, ``detect_corrupt`` (default) sends ``corrupt_score`` and ``corrupt_reason``
|
|
318
|
+
for each image. ``skip_corrupt`` leaves truncated and blank frames out of the upload and reports
|
|
319
|
+
them as skipped. Without Pillow both options do nothing.
|
|
315
320
|
"""
|
|
316
321
|
if not files:
|
|
317
322
|
raise LabelTifyError("nothing to upload")
|
|
@@ -336,6 +341,54 @@ class LabelTifyClient:
|
|
|
336
341
|
if len(dhashes) != len(images) or any(parse_dhash(h) is None for h in dhashes):
|
|
337
342
|
raise LabelTifyError(f"dhashes needs one 16-hex value per image: {len(images)} images, {len(dhashes)} hashes")
|
|
338
343
|
|
|
344
|
+
skipped_corrupt: list[dict[str, Any]] = []
|
|
345
|
+
score_by_name: dict[int, tuple[float, str] | None] = {}
|
|
346
|
+
if detect_corrupt or skip_corrupt:
|
|
347
|
+
try:
|
|
348
|
+
from .corrupt import corrupt_score
|
|
349
|
+
except ImportError:
|
|
350
|
+
detect_corrupt = False
|
|
351
|
+
else:
|
|
352
|
+
image_positions = [i for i, f in enumerate(files) if not _is_label(f["filename"])]
|
|
353
|
+
drop: set[int] = set()
|
|
354
|
+
for n, pos in enumerate(image_positions):
|
|
355
|
+
try:
|
|
356
|
+
scored = corrupt_score(files[pos]["data"])
|
|
357
|
+
except Exception:
|
|
358
|
+
score_by_name[n] = None
|
|
359
|
+
continue
|
|
360
|
+
if skip_corrupt and scored[1] in ("truncated", "blank"):
|
|
361
|
+
drop.add(pos)
|
|
362
|
+
skipped_corrupt.append(
|
|
363
|
+
{"filename": files[pos]["filename"], "status": "skipped", "message": scored[1]}
|
|
364
|
+
)
|
|
365
|
+
continue
|
|
366
|
+
score_by_name[n] = scored
|
|
367
|
+
if drop:
|
|
368
|
+
kept_scores: list[tuple[float, str] | None] = []
|
|
369
|
+
kept_hashes: list[str] = []
|
|
370
|
+
img_n = 0
|
|
371
|
+
new_files: list[UploadFile] = []
|
|
372
|
+
for pos, f in enumerate(files):
|
|
373
|
+
if _is_label(f["filename"]):
|
|
374
|
+
new_files.append(f)
|
|
375
|
+
continue
|
|
376
|
+
if pos in drop:
|
|
377
|
+
img_n += 1
|
|
378
|
+
continue
|
|
379
|
+
new_files.append(f)
|
|
380
|
+
kept_scores.append(score_by_name.get(img_n))
|
|
381
|
+
if dhashes is not None:
|
|
382
|
+
kept_hashes.append(dhashes[img_n])
|
|
383
|
+
img_n += 1
|
|
384
|
+
files = new_files
|
|
385
|
+
images = [f for f in files if not _is_label(f["filename"])]
|
|
386
|
+
score_by_name = {n: kept_scores[n] for n in range(len(kept_scores))}
|
|
387
|
+
if dhashes is not None:
|
|
388
|
+
dhashes = kept_hashes
|
|
389
|
+
if not files:
|
|
390
|
+
return {"items": [], "files": skipped_corrupt, "imported": None, "requests": 0}
|
|
391
|
+
|
|
339
392
|
boundary = f"----labeltify{uuid.uuid4().hex}"
|
|
340
393
|
parts: list[bytes] = []
|
|
341
394
|
|
|
@@ -365,11 +418,19 @@ class LabelTifyClient:
|
|
|
365
418
|
hashes = iter(dhashes or [])
|
|
366
419
|
for f in files:
|
|
367
420
|
file(f["filename"], f["content_type"], f["data"])
|
|
368
|
-
# The server pairs dhash fields with image parts in order; fields after the files are fine.
|
|
369
|
-
for _ in images:
|
|
421
|
+
# The server pairs dhash and corrupt fields with image parts in order; fields after the files are fine.
|
|
422
|
+
for n, _ in enumerate(images):
|
|
370
423
|
h = next(hashes, None)
|
|
371
424
|
if h:
|
|
372
425
|
field("dhash", h.lower())
|
|
426
|
+
scored = score_by_name.get(n) if detect_corrupt else None
|
|
427
|
+
if detect_corrupt and any(score_by_name.get(i) is not None for i in range(len(images))):
|
|
428
|
+
if scored is None:
|
|
429
|
+
field("corrupt_score", "")
|
|
430
|
+
field("corrupt_reason", "")
|
|
431
|
+
else:
|
|
432
|
+
field("corrupt_score", str(scored[0]))
|
|
433
|
+
field("corrupt_reason", scored[1])
|
|
373
434
|
parts.append(f"--{boundary}--\r\n".encode())
|
|
374
435
|
result = self._request(
|
|
375
436
|
"POST",
|
|
@@ -378,12 +439,13 @@ class LabelTifyClient:
|
|
|
378
439
|
f"multipart/form-data; boundary={boundary}",
|
|
379
440
|
)
|
|
380
441
|
body = result if isinstance(result, dict) else {}
|
|
381
|
-
|
|
442
|
+
report: UploadReport = {
|
|
382
443
|
"items": list(body.get("items") or []),
|
|
383
|
-
"files": list(body.get("files") or []),
|
|
444
|
+
"files": skipped_corrupt + list(body.get("files") or []),
|
|
384
445
|
"imported": body.get("import"),
|
|
385
446
|
"requests": 1,
|
|
386
447
|
}
|
|
448
|
+
return report
|
|
387
449
|
|
|
388
450
|
def upload_directory(
|
|
389
451
|
self,
|
|
@@ -397,6 +459,8 @@ class LabelTifyClient:
|
|
|
397
459
|
pipeline: str | None = None,
|
|
398
460
|
trigger: str | None = None,
|
|
399
461
|
compute_dhash: bool = False,
|
|
462
|
+
detect_corrupt: bool = True,
|
|
463
|
+
skip_corrupt: bool = False,
|
|
400
464
|
max_request_bytes: int | None = None,
|
|
401
465
|
max_request_images: int | None = None,
|
|
402
466
|
workers: int = DEFAULT_WORKERS,
|
|
@@ -554,6 +618,8 @@ class LabelTifyClient:
|
|
|
554
618
|
camera=camera,
|
|
555
619
|
pipeline=pipeline,
|
|
556
620
|
trigger=trigger,
|
|
621
|
+
detect_corrupt=detect_corrupt,
|
|
622
|
+
skip_corrupt=skip_corrupt,
|
|
557
623
|
max_request_bytes=max_request_bytes,
|
|
558
624
|
max_request_images=max_request_images,
|
|
559
625
|
)
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Cheap corrupt-frame heuristic. Same thresholds as `.corrupt-ref/detect.py`.
|
|
2
|
+
|
|
3
|
+
Pillow only (``pip install labeltify[images]``). No numpy. A 240 px wide thumbnail.
|
|
4
|
+
``corrupt_score(data) -> (score, reason)``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import io
|
|
10
|
+
|
|
11
|
+
TARGET_W = 240
|
|
12
|
+
BLANK_STD_SOFT, BLANK_STD_HARD = 8.0, 4.0
|
|
13
|
+
SMEAR_ROW_DIFF = 0.6
|
|
14
|
+
SMEAR_REL = 0.08
|
|
15
|
+
SMEAR_SOFT, SMEAR_HARD = 0.06, 0.15
|
|
16
|
+
GREY_TOL = 3
|
|
17
|
+
FLAT_GRAD = 0.75
|
|
18
|
+
CODEC_SOFT, CODEC_HARD = 0.02, 0.05
|
|
19
|
+
TEAR_REL = 2.5
|
|
20
|
+
TEAR_GAIN = 0.5
|
|
21
|
+
TEAR_MAXSHIFT = 96
|
|
22
|
+
THRESHOLD = 0.5
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _ramp(x: float, soft: float, hard: float) -> float:
|
|
26
|
+
t = (x - soft) / (hard - soft)
|
|
27
|
+
return float(min(1.0, max(0.0, t)))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _median(values: list[float]) -> float:
|
|
31
|
+
if not values:
|
|
32
|
+
return 0.0
|
|
33
|
+
ordered = sorted(values)
|
|
34
|
+
n = len(ordered)
|
|
35
|
+
mid = n // 2
|
|
36
|
+
if n % 2:
|
|
37
|
+
return float(ordered[mid])
|
|
38
|
+
return (ordered[mid - 1] + ordered[mid]) / 2.0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _thumb(im):
|
|
42
|
+
from PIL import Image
|
|
43
|
+
|
|
44
|
+
if im.width >= 2 * TARGET_W:
|
|
45
|
+
im = im.reduce(max(1, im.width // TARGET_W))
|
|
46
|
+
if im.width != TARGET_W:
|
|
47
|
+
im = im.resize((TARGET_W, max(8, round(im.height * TARGET_W / im.width))), Image.BILINEAR)
|
|
48
|
+
return im.convert("RGB")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _decode(data: bytes, allow_truncated: bool):
|
|
52
|
+
from PIL import Image, ImageFile
|
|
53
|
+
|
|
54
|
+
previous = ImageFile.LOAD_TRUNCATED_IMAGES
|
|
55
|
+
if allow_truncated:
|
|
56
|
+
ImageFile.LOAD_TRUNCATED_IMAGES = True
|
|
57
|
+
try:
|
|
58
|
+
im = Image.open(io.BytesIO(data))
|
|
59
|
+
if im.format == "JPEG":
|
|
60
|
+
im.draft("RGB", (TARGET_W, 1))
|
|
61
|
+
im.load()
|
|
62
|
+
return im
|
|
63
|
+
finally:
|
|
64
|
+
if allow_truncated:
|
|
65
|
+
ImageFile.LOAD_TRUNCATED_IMAGES = previous
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _load(data: bytes) -> tuple[list[tuple[int, int, int]], int, int, bool]:
|
|
69
|
+
truncated = data[:2] == b"\xff\xd8" and not data[-32:].rstrip(b"\x00\r\n ").endswith(b"\xff\xd9")
|
|
70
|
+
try:
|
|
71
|
+
im = _decode(data, False)
|
|
72
|
+
except OSError:
|
|
73
|
+
truncated = True
|
|
74
|
+
im = _decode(data, True)
|
|
75
|
+
thumb = _thumb(im)
|
|
76
|
+
w, h = thumb.size
|
|
77
|
+
pixels = list(thumb.getdata())
|
|
78
|
+
return pixels, w, h, truncated
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _luma(rgb: tuple[int, int, int]) -> float:
|
|
82
|
+
return 0.299 * rgb[0] + 0.587 * rgb[1] + 0.114 * rgb[2]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _metrics(pixels: list[tuple[int, int, int]], w: int, h: int) -> dict:
|
|
86
|
+
g = [_luma(p) for p in pixels]
|
|
87
|
+
n = w * h
|
|
88
|
+
mean = sum(g) / n
|
|
89
|
+
std = (sum((v - mean) ** 2 for v in g) / n) ** 0.5
|
|
90
|
+
rd: list[float] = []
|
|
91
|
+
for y in range(h - 1):
|
|
92
|
+
row = y * w
|
|
93
|
+
nxt = row + w
|
|
94
|
+
rd.append(sum(abs(g[nxt + x] - g[row + x]) for x in range(w)) / w)
|
|
95
|
+
med = _median(rd) + 1e-6
|
|
96
|
+
rep = [v < SMEAR_ROW_DIFF or v < SMEAR_REL * med for v in rd]
|
|
97
|
+
run = len(rep)
|
|
98
|
+
for i in range(len(rep) - 1, -1, -1):
|
|
99
|
+
if not rep[i]:
|
|
100
|
+
run = len(rep) - 1 - i
|
|
101
|
+
break
|
|
102
|
+
start = h // 3
|
|
103
|
+
mid = 0
|
|
104
|
+
in_run = False
|
|
105
|
+
run_start = 0
|
|
106
|
+
slice_rep = rep[start:]
|
|
107
|
+
for i, bit in enumerate(slice_rep + [False]):
|
|
108
|
+
if bit and not in_run:
|
|
109
|
+
in_run = True
|
|
110
|
+
run_start = i
|
|
111
|
+
elif not bit and in_run:
|
|
112
|
+
mid = max(mid, i - run_start)
|
|
113
|
+
in_run = False
|
|
114
|
+
grey = 0
|
|
115
|
+
if h >= 2 and w >= 2:
|
|
116
|
+
for y in range(h - 1):
|
|
117
|
+
for x in range(w - 1):
|
|
118
|
+
i = y * w + x
|
|
119
|
+
if abs(g[i + 1] - g[i]) + abs(g[i + w] - g[i]) >= FLAT_GRAD:
|
|
120
|
+
continue
|
|
121
|
+
r, gg, b = pixels[i]
|
|
122
|
+
if abs(r - 128) < GREY_TOL and abs(gg - 128) < GREY_TOL and abs(b - 128) < GREY_TOL:
|
|
123
|
+
grey += 1
|
|
124
|
+
green = sum(1 for r, gg, b in pixels if gg > 90 and r < 50 and b < 50)
|
|
125
|
+
tear = 0.0
|
|
126
|
+
s = min(TEAR_MAXSHIFT, w * 2 // 5)
|
|
127
|
+
if s >= 3 and len(rd) > 6:
|
|
128
|
+
inner = list(range(3, len(rd) - 3))
|
|
129
|
+
inner.sort(key=lambda i: (rd[i], i))
|
|
130
|
+
for k in inner[-3:]:
|
|
131
|
+
if rd[k] < TEAR_REL * med:
|
|
132
|
+
continue
|
|
133
|
+
d0 = _shift_mean(g, w, k, 0)
|
|
134
|
+
best = float("inf")
|
|
135
|
+
for shift in range(-s, s + 1):
|
|
136
|
+
if -2 <= shift <= 2:
|
|
137
|
+
continue
|
|
138
|
+
val = _shift_mean(g, w, k, shift)
|
|
139
|
+
if val < best:
|
|
140
|
+
best = val
|
|
141
|
+
gain = best / (d0 + 1e-6)
|
|
142
|
+
t = _ramp(gain, TEAR_GAIN, TEAR_GAIN / 2)
|
|
143
|
+
if t > tear:
|
|
144
|
+
tear = t
|
|
145
|
+
return {
|
|
146
|
+
"std": std,
|
|
147
|
+
"smear_frac": max(run, mid) / h if h else 0.0,
|
|
148
|
+
"grey_frac": grey / n,
|
|
149
|
+
"green_frac": green / n,
|
|
150
|
+
"tear": tear,
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _shift_mean(g: list[float], w: int, k: int, shift: int) -> float:
|
|
155
|
+
a0 = k * w
|
|
156
|
+
b0 = (k + 1) * w
|
|
157
|
+
total = 0.0
|
|
158
|
+
count = 0
|
|
159
|
+
for x in range(w):
|
|
160
|
+
ax = x + shift
|
|
161
|
+
if ax < 0 or ax >= w:
|
|
162
|
+
continue
|
|
163
|
+
total += abs(g[a0 + ax] - g[b0 + x])
|
|
164
|
+
count += 1
|
|
165
|
+
return total / count if count else float("inf")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _score(m: dict, truncated: bool) -> tuple[float, str]:
|
|
169
|
+
if truncated:
|
|
170
|
+
return 1.0, "truncated"
|
|
171
|
+
scores = {
|
|
172
|
+
"blank": _ramp(m["std"], BLANK_STD_SOFT, BLANK_STD_HARD),
|
|
173
|
+
"smear": _ramp(m["smear_frac"], SMEAR_SOFT, SMEAR_HARD),
|
|
174
|
+
"grey_blocks": _ramp(m["grey_frac"], CODEC_SOFT, CODEC_HARD),
|
|
175
|
+
"green_blocks": _ramp(m["green_frac"], CODEC_SOFT, CODEC_HARD),
|
|
176
|
+
"tear": m["tear"],
|
|
177
|
+
}
|
|
178
|
+
reason = max(scores, key=scores.get)
|
|
179
|
+
best = scores[reason]
|
|
180
|
+
return round(best, 3), (reason if best > 0 else "ok")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def corrupt_score(data: bytes) -> tuple[float, str]:
|
|
184
|
+
"""Score encoded image bytes. ``(score, reason)`` with score in 0..1."""
|
|
185
|
+
try:
|
|
186
|
+
pixels, w, h, truncated = _load(data)
|
|
187
|
+
except Exception:
|
|
188
|
+
return 1.0, "unreadable"
|
|
189
|
+
return _score(_metrics(pixels, w, h), truncated)
|
|
@@ -3,6 +3,7 @@ pyproject.toml
|
|
|
3
3
|
labeltify/__init__.py
|
|
4
4
|
labeltify/__main__.py
|
|
5
5
|
labeltify/client.py
|
|
6
|
+
labeltify/corrupt.py
|
|
6
7
|
labeltify/dhash.py
|
|
7
8
|
labeltify/download.py
|
|
8
9
|
labeltify.egg-info/PKG-INFO
|
|
@@ -12,4 +13,5 @@ labeltify.egg-info/entry_points.txt
|
|
|
12
13
|
labeltify.egg-info/requires.txt
|
|
13
14
|
labeltify.egg-info/top_level.txt
|
|
14
15
|
tests/test_client.py
|
|
16
|
+
tests/test_corrupt.py
|
|
15
17
|
tests/test_download.py
|
|
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "labeltify"
|
|
7
7
|
# Bump this to publish. .github/workflows/publish-python.yml releases each new version pushed to main.
|
|
8
|
-
version = "0.4.
|
|
8
|
+
version = "0.4.2"
|
|
9
9
|
description = "Upload frames and optional annotations into LabelTify with a pipeline token."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.10"
|
|
@@ -220,7 +220,7 @@ def test_bundle_sends_like_the_upload_page(api):
|
|
|
220
220
|
assert report["requests"] == 1 and report["items"] == [{"id": "img-1"}] and report["imported"] is None
|
|
221
221
|
(sent,) = uploads(api)
|
|
222
222
|
assert [f for n, f, _ in sent if n == "file"] == ["a.jpg", "b.jpg", "annotations.json"]
|
|
223
|
-
fields = {n: v for n, f, v in sent if f is None and n
|
|
223
|
+
fields = {n: v for n, f, v in sent if f is None and n not in {"dhash", "corrupt_score", "corrupt_reason"}}
|
|
224
224
|
assert fields == {"workState": b"done", "skipNearDuplicates": b"1"}
|
|
225
225
|
assert [v for n, _, v in sent if n == "dhash"] == [b"0123456789abcdef", b"fedcba9876543210"]
|
|
226
226
|
|
|
@@ -428,6 +428,44 @@ def test_compute_dhash_with_pillow(api, tmp_path):
|
|
|
428
428
|
assert len(hashes) == 2 and all(parse_dhash(h) is not None for h in hashes)
|
|
429
429
|
|
|
430
430
|
|
|
431
|
+
def test_corrupt_fields_and_skip(api):
|
|
432
|
+
from PIL import Image
|
|
433
|
+
import io
|
|
434
|
+
|
|
435
|
+
def png(color):
|
|
436
|
+
im = Image.new("RGB", (32, 32), color)
|
|
437
|
+
buf = io.BytesIO()
|
|
438
|
+
im.save(buf, format="PNG")
|
|
439
|
+
return buf.getvalue()
|
|
440
|
+
|
|
441
|
+
black = png((0, 0, 0))
|
|
442
|
+
varied = png((10, 180, 40))
|
|
443
|
+
# A flat colour may be blank; the varied one is not a full black frame.
|
|
444
|
+
# Paint a gradient so it is not blank.
|
|
445
|
+
im = Image.new("RGB", (48, 32))
|
|
446
|
+
im.putdata([((x * 5) % 256, (y * 3 + 40) % 256, (x + y) % 256) for y in range(32) for x in range(48)])
|
|
447
|
+
buf = io.BytesIO()
|
|
448
|
+
im.save(buf, format="PNG")
|
|
449
|
+
varied = buf.getvalue()
|
|
450
|
+
client = client_for(api)
|
|
451
|
+
client.upload_bundle(
|
|
452
|
+
"ds-1",
|
|
453
|
+
[
|
|
454
|
+
{"filename": "black.png", "data": black, "content_type": "image/png"},
|
|
455
|
+
{"filename": "scene.png", "data": varied, "content_type": "image/png"},
|
|
456
|
+
],
|
|
457
|
+
skip_near_duplicates=False,
|
|
458
|
+
skip_corrupt=True,
|
|
459
|
+
)
|
|
460
|
+
(sent,) = uploads(api)
|
|
461
|
+
names = [f for n, f, _ in sent if n == "file"]
|
|
462
|
+
assert names == ["scene.png"]
|
|
463
|
+
reasons = [v.decode() for n, _, v in sent if n == "corrupt_reason"]
|
|
464
|
+
assert reasons and all(r != "blank" for r in reasons)
|
|
465
|
+
scores = [n for n, _, _ in sent if n == "corrupt_score"]
|
|
466
|
+
assert scores
|
|
467
|
+
|
|
468
|
+
|
|
431
469
|
def test_no_dhash_without_asking(api, tmp_path):
|
|
432
470
|
(tmp_path / "a.jpg").write_bytes(b"\xff\xd8a")
|
|
433
471
|
client_for(api).upload_directory("ds-1", tmp_path)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Synthetic frames only. The same outcomes as packages/shared corruptScoreFromRgba."""
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
|
|
5
|
+
from PIL import Image
|
|
6
|
+
|
|
7
|
+
from labeltify.corrupt import THRESHOLD, corrupt_score
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def png(width, height, pixel) -> bytes:
|
|
11
|
+
im = Image.new("RGB", (width, height))
|
|
12
|
+
im.putdata([pixel(x, y) for y in range(height) for x in range(width)])
|
|
13
|
+
buf = io.BytesIO()
|
|
14
|
+
im.save(buf, format="PNG")
|
|
15
|
+
return buf.getvalue()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_clean_scene_is_ok():
|
|
19
|
+
data = png(240, 48, lambda x, y: ((v := (x * 3 + y * 5) % 256), (v + 40) % 256, (v + 90) % 256))
|
|
20
|
+
score, reason = corrupt_score(data)
|
|
21
|
+
assert reason == "ok"
|
|
22
|
+
assert score < THRESHOLD
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_black_frame_is_blank():
|
|
26
|
+
score, reason = corrupt_score(png(240, 32, lambda x, y: (0, 0, 0)))
|
|
27
|
+
assert reason == "blank"
|
|
28
|
+
assert score >= THRESHOLD
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_bottom_row_repeat_is_smear():
|
|
32
|
+
def pixel(x, y):
|
|
33
|
+
row = 27 if y >= 28 else y
|
|
34
|
+
v = 40 + row * 6
|
|
35
|
+
return (v, 30, 180)
|
|
36
|
+
|
|
37
|
+
score, reason = corrupt_score(png(240, 40, pixel))
|
|
38
|
+
assert reason == "smear"
|
|
39
|
+
assert score >= THRESHOLD
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_grey_block():
|
|
43
|
+
def pixel(x, y):
|
|
44
|
+
if 20 <= x < 200 and 8 <= y < 40:
|
|
45
|
+
return (128, 128, 128)
|
|
46
|
+
v = (x * 7 + y * 3) % 256
|
|
47
|
+
return (v, (v + 20) % 256, (v + 70) % 256)
|
|
48
|
+
|
|
49
|
+
score, reason = corrupt_score(png(240, 60, pixel))
|
|
50
|
+
assert reason == "grey_blocks"
|
|
51
|
+
assert score >= THRESHOLD
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_green_block():
|
|
55
|
+
def pixel(x, y):
|
|
56
|
+
if 10 <= x < 200 and 8 <= y < 40:
|
|
57
|
+
return (0, 140, 0)
|
|
58
|
+
v = (x * 7 + y * 3) % 256
|
|
59
|
+
return (v, (v + 20) % 256, (v + 70) % 256)
|
|
60
|
+
|
|
61
|
+
score, reason = corrupt_score(png(240, 60, pixel))
|
|
62
|
+
assert reason == "green_blocks"
|
|
63
|
+
assert score >= THRESHOLD
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_shift_seam_is_tear():
|
|
67
|
+
seam = 20
|
|
68
|
+
|
|
69
|
+
def pixel(x, y):
|
|
70
|
+
xx = x + 8 if y == seam + 1 else x
|
|
71
|
+
yy = seam if y == seam + 1 else y
|
|
72
|
+
v = (xx * 3 + yy) % 256
|
|
73
|
+
return (v, v, v)
|
|
74
|
+
|
|
75
|
+
score, reason = corrupt_score(png(240, 48, pixel))
|
|
76
|
+
assert reason == "tear"
|
|
77
|
+
assert score >= THRESHOLD
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_jpeg_missing_end_marker_is_truncated():
|
|
81
|
+
im = Image.new("RGB", (32, 32), (20, 40, 60))
|
|
82
|
+
buf = io.BytesIO()
|
|
83
|
+
im.save(buf, format="JPEG")
|
|
84
|
+
raw = buf.getvalue()
|
|
85
|
+
assert raw.endswith(b"\xff\xd9")
|
|
86
|
+
score, reason = corrupt_score(raw[:-2] + b"\x00\x11")
|
|
87
|
+
assert reason == "truncated"
|
|
88
|
+
assert score == 1.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|