screengraft 0.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,677 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ screengraft — M1a: no-ML screen-quad detection.
4
+
5
+ Finds the four corners of a device screen in a photo, so `warp.py` has a quad
6
+ to warp into. Deterministic and dependency-light: thresholding, morphology and
7
+ line fitting — no model, no download, no network.
8
+
9
+ The pipeline (proven on a real leaning-phone render, 3 Sep 2026):
10
+
11
+ 1. Sweep candidate tone bands over the grayscale histogram. A device screen
12
+ is a large region whose pixels sit in a narrow tone band, distinct from
13
+ bezel and background — so instead of guessing one threshold, try every
14
+ band and let the scoring decide.
15
+ 2. For each band: close (fills UI text and icons back into the screen body),
16
+ then a large open (kills speckled shadow that would otherwise bridge the
17
+ screen to the background), then take the largest connected component.
18
+ 3. Score that blob on how much it looks like a quadrilateral — area, how
19
+ completely it fills its own convex 4-gon, and 4-point approximability.
20
+ Best-scoring band wins.
21
+ 4. Fit a line to the straight middle stretch of each of the four edges and
22
+ intersect adjacent lines. This is the step that matters: rounded screen
23
+ corners pull a naive corner estimate inward, and intersecting the straight
24
+ edges recovers the true (virtual) corners the homography needs.
25
+
26
+ Fails honestly. When the screen's tone isn't separable from its surroundings
27
+ (a bright UI photographed against a light wall), no band scores well; the
28
+ script says so and exits non-zero rather than emitting a confident wrong quad.
29
+ That failure is the case SAM 2 (M4) is meant to earn its keep on.
30
+
31
+ Detection never warps anything. It writes corners and an overlay for a human
32
+ to confirm — `warp.py` still requires corners passed explicitly, so an
33
+ unconfirmed guess can never reach the composite.
34
+
35
+ Usage:
36
+ python3 detect.py --photo photo.jpg --out-corners corners.json \
37
+ [--out-overlay overlay.png] [--out-zooms DIR] [--tone LO,HI]
38
+ """
39
+
40
+ import argparse
41
+ import json
42
+ import os
43
+ import sys
44
+
45
+ import cv2
46
+ import numpy as np
47
+
48
+ # Morphology kernel sizes are expressed as a fraction of the image's short
49
+ # side, so behaviour doesn't change with photo resolution.
50
+ CLOSE_FRAC = 0.008 # ~10px on a 1200px-short-side photo: fills UI text
51
+ OPEN_FRAC = 0.023 # ~41px on a 1792px-short-side photo: severs shadow bridges
52
+ EDGE_MIDDLE = 0.70 # fit each edge line on its straight middle 70%
53
+ MIN_AREA_FRAC = 0.01
54
+ MAX_AREA_FRAC = 0.60
55
+ MIN_FILL = 0.60 # the source blob must fill this much of the final quad
56
+ MIN_SIDE_RATIO = 0.08 # reject slivers: shortest side vs longest, after perspective
57
+ # A screen fills MOST of the body it sits in; content drawn on a screen is a
58
+ # small part of it. That one ratio separates "step inward to the screen" from
59
+ # "don't step into a panel", and it arbitrates between the two detectors too.
60
+ NEST_FLOOR = 0.55
61
+
62
+
63
+ def _odd(n: int) -> int:
64
+ n = max(3, int(round(float(n))))
65
+ return n if n % 2 else n + 1
66
+
67
+
68
+ def blob_for_band(gray: np.ndarray, lo: int, hi: int, close_k: int, open_k: int):
69
+ """Mask -> morphology -> largest connected component. Returns its contour."""
70
+ mask = cv2.inRange(gray, lo, hi)
71
+ mask = cv2.morphologyEx(
72
+ mask, cv2.MORPH_CLOSE,
73
+ cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (close_k, close_k)),
74
+ )
75
+ mask = cv2.morphologyEx(
76
+ mask, cv2.MORPH_OPEN,
77
+ cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (open_k, open_k)),
78
+ )
79
+ # CHAIN_APPROX_NONE, deliberately: the edge refinement fits a line to each
80
+ # side, and SIMPLE compresses straight runs down to their endpoints, which
81
+ # leaves fitLine with nothing to fit.
82
+ contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
83
+ if not contours:
84
+ return None
85
+ return max(contours, key=cv2.contourArea)
86
+
87
+
88
+ def score_contour(contour, img_area: float):
89
+ """How much does this blob look like a flat screen? Higher is better."""
90
+ area = cv2.contourArea(contour)
91
+ if area < MIN_AREA_FRAC * img_area or area > MAX_AREA_FRAC * img_area:
92
+ return 0.0, None
93
+ quad = approx_quad(contour)
94
+ if quad is None:
95
+ return 0.0, None
96
+ quad_area = cv2.contourArea(quad.astype(np.float32))
97
+ if quad_area <= 0:
98
+ return 0.0, None
99
+ # Fill ratio: a real screen fills its own quad almost completely. A shadow
100
+ # or a wall patch is ragged and fills far less.
101
+ fill = min(area / quad_area, 1.0)
102
+ return (fill ** 3) * (area / img_area), quad
103
+
104
+
105
+ # Two "edge support" arbiters — score each candidate quad by how much of its
106
+ # perimeter sits on a Canny edge, then by whether the gradient forms a local
107
+ # ridge across it — were built and measured here on 3 Sep 2026, and both
108
+ # dropped. Numbers, correct answer first:
109
+ #
110
+ # case Canny support ridge support
111
+ # synthetic fixture 0.023 vs 1.000 0.927 vs 0.999 (both prefer wrong)
112
+ # gradient screen -- 0.932 correct (works)
113
+ # real phone photo 0.728 vs 0.339 0.082 vs 0.172 (ridge prefers wrong)
114
+ # real mockup 0.215 correct 0.062 correct (below any usable floor)
115
+ #
116
+ # Edge strength is not a usable arbiter in this domain, for the reason an earlier finding
117
+ # already found from the other direction: a screen's own boundary (glass
118
+ # against a dark bezel) is routinely WEAKER than the device silhouette beside
119
+ # it, and on real photographs both are soft enough that any credibility floor
120
+ # strict enough to reject a wrong quad also rejects the right one. Geometry
121
+ # generalised; photometry did not. detect() arbitrates on nesting instead —
122
+ # the same rule pick_innermost() already uses.
123
+
124
+
125
+ def score_edge_contour(contour, img_area: float):
126
+ """Scoring for the Canny path, where score_contour()'s assumptions break.
127
+
128
+ Two differences that matter. Canny traces a closed *ring* around each
129
+ edge, so the contour's own area is an artifact of the ring, not of the
130
+ region — score the QUAD's area instead, which is what gets returned.
131
+ And the fill ratio is correspondingly noisy (it can exceed 1), so it
132
+ weighs in linearly rather than cubed; cubing it once cost the real screen
133
+ the win against a wave-shaped gradient boundary drawn inside that screen
134
+ (fill 0.86 vs a ring artifact's 1.0).
135
+ """
136
+ quad = approx_quad(contour)
137
+ if quad is None:
138
+ return 0.0, None
139
+ quad_area = float(cv2.contourArea(order_quad(quad).astype(np.float32)))
140
+ if quad_area < MIN_AREA_FRAC * img_area or quad_area > MAX_AREA_FRAC * img_area:
141
+ return 0.0, None
142
+ fill = min(float(cv2.contourArea(contour)) / max(quad_area, 1e-6), 1.0)
143
+ return fill * (quad_area / img_area), quad
144
+
145
+
146
+ def approx_quad(contour):
147
+ """Reduce a contour to 4 convex points, or None."""
148
+ hull = cv2.convexHull(contour)
149
+ peri = cv2.arcLength(hull, True)
150
+ for eps in np.arange(0.01, 0.12, 0.005):
151
+ approx = cv2.approxPolyDP(hull, eps * peri, True)
152
+ if len(approx) == 4 and cv2.isContourConvex(approx):
153
+ return approx.reshape(4, 2).astype(np.float64)
154
+ return None
155
+
156
+
157
+ def order_quad(pts: np.ndarray) -> np.ndarray:
158
+ """Order 4 points TL, TR, BR, BL as they appear in the image."""
159
+ c = pts.mean(axis=0)
160
+ ang = np.arctan2(pts[:, 1] - c[1], pts[:, 0] - c[0])
161
+ pts = pts[np.argsort(ang)] # counter-clockwise in image coords
162
+ start = int(np.argmin(pts.sum(axis=1))) # closest to the image's top-left
163
+ return np.roll(pts, -start, axis=0)
164
+
165
+
166
+ def refine_corners(contour, quad: np.ndarray):
167
+ """
168
+ Re-derive corners by intersecting fitted edge lines.
169
+
170
+ Rounded screen corners bend the contour inward, so the polygon's own
171
+ vertices sit inside the true corner. Each edge's straight middle stretch,
172
+ extended, meets its neighbour at the corner the homography actually wants.
173
+
174
+ Every contour point is assigned to whichever of the four edges it lies
175
+ nearest, then only the middle stretch of each edge is fitted — which drops
176
+ the rounded corners without needing a distance threshold to tune. The fit
177
+ is Huber rather than least-squares so a bump in the silhouette (a shadow
178
+ nick, a finger) pulls the line far less than it otherwise would.
179
+
180
+ Returns (corners, refined) — `refined` False means it declined and handed
181
+ back the polygon's own corners, which the caller should surface rather
182
+ than quietly present as a refined result.
183
+ """
184
+ pts = contour.reshape(-1, 2).astype(np.float64)
185
+ if len(pts) < 40:
186
+ return quad, False
187
+
188
+ # Distance from every point to every edge; nearest edge wins the point.
189
+ dists, projections = [], []
190
+ for i in range(4):
191
+ a, b = quad[i], quad[(i + 1) % 4]
192
+ ab = b - a
193
+ length = float(np.linalg.norm(ab))
194
+ if length < 1e-6:
195
+ return quad, False
196
+ u = ab / length
197
+ rel = pts - a
198
+ dists.append(np.abs(u[0] * rel[:, 1] - u[1] * rel[:, 0]))
199
+ projections.append((rel @ ab) / (length ** 2))
200
+ nearest = np.argmin(np.vstack(dists), axis=0)
201
+
202
+ margin = (1.0 - EDGE_MIDDLE) / 2.0
203
+ lines = []
204
+ for i in range(4):
205
+ t = projections[i]
206
+ sel = pts[(nearest == i) & (t > margin) & (t < 1.0 - margin)]
207
+ if len(sel) < 10:
208
+ return quad, False
209
+ vx, vy, x0, y0 = cv2.fitLine(
210
+ sel.astype(np.float32), cv2.DIST_HUBER, 0, 0.01, 0.01
211
+ ).ravel()
212
+ lines.append((float(vx), float(vy), float(x0), float(y0)))
213
+
214
+ corners = []
215
+ for i in range(4):
216
+ # Corner i is where edge (i-1) meets edge i.
217
+ vx1, vy1, x1, y1 = lines[(i - 1) % 4]
218
+ vx2, vy2, x2, y2 = lines[i]
219
+ denom = vx1 * vy2 - vy1 * vx2
220
+ if abs(denom) < 1e-9:
221
+ return quad, False # parallel edges: refinement is meaningless
222
+ t = ((x2 - x1) * vy2 - (y2 - y1) * vx2) / denom
223
+ corners.append([x1 + t * vx1, y1 + t * vy1])
224
+ corners = np.array(corners, dtype=np.float64)
225
+
226
+ # A refinement that moves a corner further than a quarter of the quad's
227
+ # size is not a refinement; something upstream was wrong.
228
+ span = float(np.linalg.norm(quad[2] - quad[0]))
229
+ if np.max(np.linalg.norm(corners - quad, axis=1)) > 0.25 * span:
230
+ return quad, False
231
+ return corners, True
232
+
233
+
234
+ def draw_overlay(photo: np.ndarray, corners: np.ndarray) -> np.ndarray:
235
+ out = photo.copy()
236
+ poly = corners.astype(np.int32)
237
+ cv2.polylines(out, [poly], True, (0, 255, 0), 2, cv2.LINE_AA)
238
+ for (x, y), label in zip(corners, ["TL", "TR", "BR", "BL"], strict=True):
239
+ p = (int(round(x)), int(round(y)))
240
+ cv2.circle(out, p, 9, (0, 0, 255), 2, cv2.LINE_AA)
241
+ cv2.circle(out, p, 2, (0, 0, 255), -1, cv2.LINE_AA)
242
+ cv2.putText(out, label, (p[0] + 12, p[1] - 12),
243
+ cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2, cv2.LINE_AA)
244
+ return out
245
+
246
+
247
+ def write_zooms(photo: np.ndarray, corners: np.ndarray, out_dir: str, box: int = 140):
248
+ """Corner close-ups — what a human needs to actually confirm a quad."""
249
+ os.makedirs(out_dir, exist_ok=True)
250
+ h, w = photo.shape[:2]
251
+ paths = []
252
+ overlaid = draw_overlay(photo, corners)
253
+ for (x, y), label in zip(corners, ["TL", "TR", "BR", "BL"], strict=True):
254
+ cx, cy = int(round(x)), int(round(y))
255
+ x0, y0 = max(0, cx - box), max(0, cy - box)
256
+ x1, y1 = min(w, cx + box), min(h, cy + box)
257
+ crop = overlaid[y0:y1, x0:x1]
258
+ if crop.size == 0:
259
+ continue
260
+ crop = cv2.resize(crop, None, fx=2, fy=2, interpolation=cv2.INTER_NEAREST)
261
+ path = os.path.join(out_dir, f"zoom_{label}.png")
262
+ cv2.imwrite(path, crop, [cv2.IMWRITE_PNG_COMPRESSION, 9])
263
+ paths.append(path)
264
+ return paths
265
+
266
+
267
+ def quad_contains(outer: np.ndarray, inner: np.ndarray, margin: float = 2.0) -> bool:
268
+ """Is `inner` wholly inside `outer`?"""
269
+ poly = outer.astype(np.float32)
270
+ return all(
271
+ cv2.pointPolygonTest(poly, (float(x), float(y)), True) > margin
272
+ for x, y in inner
273
+ )
274
+
275
+
276
+ def overlap_frac(inner: np.ndarray, outer: np.ndarray) -> float:
277
+ """How much of `inner`'s area falls inside `outer` (0..1)."""
278
+ a = float(cv2.contourArea(inner.astype(np.float32)))
279
+ if a <= 0:
280
+ return 0.0
281
+ # The first return is the intersection area, deliberately discarded: it is
282
+ # recomputed from `region` below so this function has exactly one notion of
283
+ # area, measured the same way as `a`.
284
+ _, region = cv2.intersectConvexConvex(inner.astype(np.float32),
285
+ outer.astype(np.float32))
286
+ if region is None or len(region) < 3:
287
+ return 0.0
288
+ return float(cv2.contourArea(region.astype(np.float32))) / a
289
+
290
+
291
+ def pick_innermost(candidates, best):
292
+ """
293
+ A device photo offers more than one screen-shaped region: the glass screen,
294
+ and the bezel or body it sits in. They nest, and the outer one wins on
295
+ area alone — so once a best candidate is found, step inward while a
296
+ comparably screen-like quad sits wholly inside it. The innermost such
297
+ region is the screen; that's the one the UI has to be warped into.
298
+
299
+ The nesting floor is deliberately high. A screen fills most of the body
300
+ it sits in, so a genuine step inward barely shrinks. Content *drawn on*
301
+ the screen — a panel, a card, a gradient band — is much smaller relative
302
+ to its parent, and stepping into it is a miss, not a refinement. At the
303
+ old 0.35 floor a wave-shaped gradient at 49% of its screen was eligible;
304
+ at 0.55 it isn't (an earlier finding, 3 Sep 2026).
305
+ """
306
+ current = best
307
+ for _ in range(4): # screen inside bezel inside body: a few steps is plenty
308
+ c_area = cv2.contourArea(current[1].astype(np.float32))
309
+ inner = [
310
+ c for c in candidates
311
+ if c is not current
312
+ and c[0] >= 0.25 * current[0]
313
+ and quad_contains(current[1], c[1])
314
+ and NEST_FLOOR * c_area <= cv2.contourArea(c[1].astype(np.float32)) < c_area
315
+ ]
316
+ if not inner:
317
+ return current
318
+ # Largest of the nested ones: the screen, not a panel drawn on it.
319
+ current = max(inner, key=lambda c: cv2.contourArea(c[1].astype(np.float32)))
320
+ return current
321
+
322
+
323
+ def measure_corner_radius(contour, corners: np.ndarray):
324
+ """Estimate the screen's corner radius from the mask outline.
325
+
326
+ For a rounded corner of radius r, the outline's nearest point to the
327
+ virtual (sharp) corner lies on the bisector at distance r*(sqrt(2)-1).
328
+ So r ~= d / 0.4142 per corner. Perspective skews this a little, so we
329
+ report the median over the four corners, express it as a fraction of
330
+ the screen's width (so warp.py can turn it into screenshot pixels), and
331
+ flag confidence from how well the four corners agree.
332
+ """
333
+ pts = contour.reshape(-1, 2).astype(np.float64)
334
+ top = np.linalg.norm(corners[1] - corners[0])
335
+ bottom = np.linalg.norm(corners[2] - corners[3])
336
+ width = float((top + bottom) / 2.0)
337
+ per = []
338
+ for c in corners:
339
+ d = float(np.min(np.linalg.norm(pts - c, axis=1)))
340
+ per.append(d / (np.sqrt(2.0) - 1.0))
341
+ per = np.array(per)
342
+ r = float(np.median(per))
343
+ spread = float((per.max() - per.min()) / max(r, 1e-6))
344
+ confident = bool(r > 2.0 and spread < 0.5)
345
+ return {
346
+ "photo_px": round(r, 1),
347
+ "frac_of_width": round(r / width, 4) if width > 0 else 0.0,
348
+ "per_corner_px": [round(float(v), 1) for v in per],
349
+ "confident": confident,
350
+ "note": ("Median of four per-corner estimates; spread %.0f%%." % (spread * 100))
351
+ + (" Good agreement." if confident else
352
+ " Poor agreement or near-square corners: offer device presets instead."),
353
+ }
354
+
355
+
356
+ # A local, per-handle version of refine_corners() below — "snap this one
357
+ # corner to its nearest edge on release" — was prototyped and measured
358
+ # against the real reference photo (see build-brief.md, "an earlier finding" section): mean
359
+ # error did not improve over the rough dragged position (19.6px unrefined vs
360
+ # 21.5px refined), because the true bezel/screen edge is often LOWER local
361
+ # contrast than a nearby wrong edge (e.g. bezel against a bright background),
362
+ # so "strongest/nearest gradient" reliably locks onto the wrong one. Dropped
363
+ # rather than wired into the UI. detect()'s global refine_corners() below is
364
+ # a different, more reliable case — it fits all four edges from one already-
365
+ # segmented contour, so it isn't picking between competing nearby edges.
366
+
367
+
368
+ def validate_quad(corners: np.ndarray, contour, img_area: float):
369
+ """Is this *final* quad plausibly a screen? Returns (ok, reason).
370
+
371
+ The area gate in score_contour() runs against the CONTOUR, before
372
+ approxPolyDP and refine_corners() reshape it — so a contour that scrapes
373
+ past the floor can still yield a final quad far below it. That is exactly
374
+ how a 0.72%-of-image sliver around a watermark was once returned as a
375
+ confident detection (3 Sep 2026, an earlier finding). Everything returned to a caller
376
+ goes through here.
377
+ """
378
+ area = float(cv2.contourArea(corners.astype(np.float32)))
379
+ frac = area / img_area
380
+ if frac < MIN_AREA_FRAC:
381
+ return False, ("final quad covers only %.2f%% of the photo (floor %.0f%%) "
382
+ "— too small to be the screen" % (frac * 100, MIN_AREA_FRAC * 100))
383
+ if frac > MAX_AREA_FRAC:
384
+ return False, ("final quad covers %.0f%% of the photo (ceiling %.0f%%) "
385
+ "— that's the scene, not a screen" % (frac * 100, MAX_AREA_FRAC * 100))
386
+ if not cv2.isContourConvex(corners.astype(np.float32).reshape(-1, 1, 2)):
387
+ return False, "final quad is not convex — a plane in perspective always is"
388
+ sides = [float(np.linalg.norm(corners[(i + 1) % 4] - corners[i])) for i in range(4)]
389
+ if min(sides) < MIN_SIDE_RATIO * max(sides):
390
+ return False, ("final quad is a sliver (shortest side %.0f%% of the longest) "
391
+ % (100 * min(sides) / max(sides)))
392
+ if contour is not None:
393
+ fill = float(cv2.contourArea(contour)) / max(area, 1e-6)
394
+ if fill < MIN_FILL:
395
+ return False, ("source blob fills only %.0f%% of the final quad "
396
+ "(floor %.0f%%) — ragged, not a screen"
397
+ % (fill * 100, MIN_FILL * 100))
398
+ return True, ""
399
+
400
+
401
+ def _finalize(candidates, img_area: float, refine: bool = True):
402
+ """Best candidate that survives refinement AND validation.
403
+
404
+ Walks candidates best-score-first rather than trusting the top one: a
405
+ high-scoring blob whose refined quad fails the plausibility gate is a
406
+ miss, not a result, and the next candidate deserves a look before the
407
+ detector gives up.
408
+
409
+ `refine` is off for the Canny path. refine_corners() assumes the contour
410
+ is a filled region's silhouette, where each side has one long straight
411
+ run to fit. A Canny contour is a ring tracing both sides of an edge, with
412
+ content edges caught inside it, so the per-edge line fits pick up the
413
+ wrong points: measured on the gradient-screen mockup it pushed one corner
414
+ 78px off an otherwise correct quad . The polygon approximation of a
415
+ Canny boundary is already on the edge, so there is nothing to recover.
416
+ """
417
+ rejected = []
418
+ for cand in sorted(candidates, key=lambda c: c[0], reverse=True):
419
+ score, quad, contour, tag = pick_innermost(candidates, cand)
420
+ if refine:
421
+ refined, did_refine = refine_corners(contour, quad)
422
+ else:
423
+ refined, did_refine = quad, False
424
+ corners = order_quad(refined)
425
+ ok, why = validate_quad(corners, contour, img_area)
426
+ if ok:
427
+ return {
428
+ "corners": [[round(float(x), 1), round(float(y), 1)] for x, y in corners],
429
+ "score": round(float(score), 5),
430
+ "edge_refined": did_refine,
431
+ "corner_radius": measure_corner_radius(contour, corners),
432
+ "_corners_np": corners,
433
+ "_tag": tag,
434
+ "rejected": rejected,
435
+ }
436
+ rejected.append({"score": round(float(score), 5), "tag": tag, "why": why})
437
+ return None
438
+
439
+
440
+ def detect_tone(gray: np.ndarray, tone=None):
441
+ """Tone-band segmentation. Assumes the screen sits in a narrow tone band."""
442
+ h, w = gray.shape[:2]
443
+ img_area = float(h * w)
444
+ short = min(h, w)
445
+ close_k = _odd(CLOSE_FRAC * short)
446
+ open_k = _odd(OPEN_FRAC * short)
447
+
448
+ if tone is not None:
449
+ bands = [tone]
450
+ else:
451
+ # 32 bins over the 0-255 range; try each bin, and each adjacent pair
452
+ # (a screen showing a gradient can straddle two bins).
453
+ edges = [int(round(i * 256 / 32)) for i in range(33)]
454
+ bands = [(edges[i], edges[i + 1] - 1) for i in range(32)]
455
+ bands += [(edges[i], edges[i + 2] - 1) for i in range(31)]
456
+
457
+ candidates = []
458
+ for lo, hi in bands:
459
+ contour = blob_for_band(gray, lo, hi, close_k, open_k)
460
+ if contour is None:
461
+ continue
462
+ score, quad = score_contour(contour, img_area)
463
+ if score > 0 and quad is not None:
464
+ # Prefer a narrow band. A screen is tonally uniform; a band wide
465
+ # enough to also swallow the phone's shadowed body scores well on
466
+ # area but produces edges that follow the body, not the glass.
467
+ # Measured on the reference photo: this alone cut the worst-corner
468
+ # error from 110px to 66px.
469
+ score *= (16.0 / (hi - lo + 1)) ** 0.25
470
+ candidates.append((score, order_quad(quad), contour, (lo, hi)))
471
+
472
+ res = _finalize(candidates, img_area)
473
+ if res is None:
474
+ return None
475
+ band = res.pop("_tag")
476
+ res["method"] = "tone"
477
+ res["band"] = [int(band[0]), int(band[1])]
478
+ return res
479
+
480
+
481
+ def detect_edges(gray: np.ndarray):
482
+ """Canny-and-quad detection — the document-scanner path.
483
+
484
+ Tone banding assumes a near-uniform screen, which breaks the moment the
485
+ screen shows real UI: a gradient wallpaper never lands inside one band,
486
+ so the sweep finds nothing (or worse, something else — an earlier finding, 3 Sep 2026).
487
+ This keys on the *boundary* instead of the fill, so screen content is
488
+ irrelevant; all it needs is contrast between glass and bezel, which a
489
+ device photo has by construction.
490
+ """
491
+ h, w = gray.shape[:2]
492
+ img_area = float(h * w)
493
+ short = min(h, w)
494
+ blur = cv2.GaussianBlur(gray, (5, 5), 0)
495
+
496
+ candidates = []
497
+ # Sweep the Canny thresholds off the image's own median rather than fixed
498
+ # numbers, then a few sigmas around it — one exposure doesn't suit both a
499
+ # bright render and a dim photo.
500
+ med = float(np.median(blur))
501
+ for sigma in (0.20, 0.33, 0.50, 0.66):
502
+ lo = int(max(0, (1.0 - sigma) * med))
503
+ hi = int(min(255, (1.0 + sigma) * med))
504
+ if hi <= lo:
505
+ continue
506
+ edges = cv2.Canny(blur, lo, hi, L2gradient=True)
507
+ # Close small gaps so a bezel outline broken by a notch or a glare
508
+ # spot still forms one closed contour.
509
+ k = _odd(0.004 * short)
510
+ edges = cv2.morphologyEx(
511
+ edges, cv2.MORPH_CLOSE,
512
+ cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k)),
513
+ )
514
+ contours, _ = cv2.findContours(edges, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
515
+ for c in contours:
516
+ score, quad = score_edge_contour(c, img_area)
517
+ if score > 0 and quad is not None:
518
+ candidates.append((score, order_quad(quad), c, (lo, hi)))
519
+
520
+ res = _finalize(candidates, img_area, refine=False)
521
+ if res is None:
522
+ return None
523
+ thr = res.pop("_tag")
524
+ res["method"] = "edge"
525
+ res["canny"] = [int(thr[0]), int(thr[1])]
526
+ return res
527
+
528
+
529
+ def detect(gray: np.ndarray, tone=None, method="auto"):
530
+ """Run both detectors; arbitrate on how the two quads nest.
531
+
532
+ The two fail on opposite things. Tone banding needs a tonally uniform
533
+ screen: it finds nothing when the screen shows real UI, or — worse — a
534
+ horizontal slab of a gradient wallpaper, which looks like a perfectly
535
+ good screen. Edge detection needs only a visible boundary, so it handles
536
+ content-filled screens, but with nothing to tell glass from bezel it will
537
+ return the device's outer silhouette instead.
538
+
539
+ Ranking by score is meaningless (the two are computed differently), and
540
+ two photometric arbiters were built, measured and rejected — see the note
541
+ above score_edge_contour(). What does separate the cases is the same
542
+ geometry pick_innermost() already relies on: a screen fills MOST of the
543
+ body it sits in, while content drawn on a screen is a small part of it.
544
+ So when tone's quad sits inside edge's, the area ratio says which is
545
+ which — 0.79 on the fixture (screen in body: take the inner, tone), 0.41
546
+ on a gradient screen (slab on screen: take the outer, edge). When they
547
+ don't nest at all, neither is a subregion of the other and tone wins,
548
+ since its assumptions being met is itself evidence and it refines corners.
549
+
550
+ Disagreement is reported, never silently resolved — the human confirms
551
+ the corners either way.
552
+ """
553
+ results = []
554
+ if method in ("auto", "tone"):
555
+ r = detect_tone(gray, tone)
556
+ if r:
557
+ results.append(r)
558
+ if method in ("auto", "edge") and tone is None:
559
+ r = detect_edges(gray)
560
+ if r:
561
+ results.append(r)
562
+
563
+ if not results:
564
+ return None
565
+
566
+ t = next((r for r in results if r["method"] == "tone"), None)
567
+ e = next((r for r in results if r["method"] == "edge"), None)
568
+ why = ""
569
+ if t and e:
570
+ tq, eq = t["_corners_np"], e["_corners_np"]
571
+ ta = float(cv2.contourArea(tq.astype(np.float32)))
572
+ ea = float(cv2.contourArea(eq.astype(np.float32)))
573
+ nested = overlap_frac(tq, eq) >= 0.90 and ta < ea
574
+ ratio = ta / ea if ea > 0 else 0.0
575
+ if nested and ratio < NEST_FLOOR:
576
+ best, why = e, ("the tone quad is only %.0f%% of the edge quad it sits "
577
+ "inside — that's content drawn on the screen, not the "
578
+ "screen" % (ratio * 100))
579
+ elif nested:
580
+ best, why = t, ("the tone quad sits inside the edge quad at %.0f%% of "
581
+ "its area — a screen inside a device body" % (ratio * 100))
582
+ else:
583
+ best, why = t, "the two quads aren't nested; the tone detector refines corners"
584
+ else:
585
+ best = t or e
586
+
587
+ if len(results) == 2:
588
+ a, b = (r["_corners_np"] for r in results)
589
+ diag = float(np.hypot(*gray.shape[:2]))
590
+ spread = float(np.max(np.linalg.norm(a - b, axis=1))) / diag
591
+ agree = bool(spread < 0.02)
592
+ best["agreement"] = {
593
+ "both_found": True,
594
+ "max_corner_gap_frac_of_diagonal": round(spread, 4),
595
+ "agree": agree,
596
+ "note": ("Both detectors independently landed on the same quad — "
597
+ "that is real evidence, not one algorithm's opinion."
598
+ if agree else
599
+ "The two detectors disagree by %.0f%% of the image diagonal. "
600
+ "Showing the %s one because %s — but check all four corners."
601
+ % (spread * 100, best["method"], why)),
602
+ }
603
+ best["agreement"]["chosen_because"] = why
604
+ other = next(r for r in results if r is not best)
605
+ best["other"] = {"method": other["method"], "corners": other["corners"]}
606
+ else:
607
+ best["agreement"] = {
608
+ "both_found": False,
609
+ "agree": False,
610
+ "note": ("Only the %s detector found anything — a first guess to "
611
+ "correct, not a measurement." % best["method"]),
612
+ }
613
+ return best
614
+
615
+
616
+ def main() -> None:
617
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
618
+ ap.add_argument("--photo", required=True)
619
+ ap.add_argument("--out-corners", required=True, help="Where to write the corners JSON")
620
+ ap.add_argument("--out-overlay", help="Optional PNG showing the detected quad on the photo")
621
+ ap.add_argument("--out-zooms", help="Optional directory for 2x corner close-ups")
622
+ ap.add_argument("--tone", help="Skip the sweep and force one band, e.g. --tone 20,40")
623
+ args = ap.parse_args()
624
+
625
+ photo = cv2.imread(args.photo, cv2.IMREAD_COLOR)
626
+ if photo is None:
627
+ sys.exit(f"error: could not read photo: {args.photo}")
628
+ gray = cv2.cvtColor(photo, cv2.COLOR_BGR2GRAY)
629
+
630
+ tone = None
631
+ if args.tone:
632
+ try:
633
+ lo, hi = (int(v) for v in args.tone.split(","))
634
+ tone = (lo, hi)
635
+ except ValueError:
636
+ sys.exit("error: --tone must be LO,HI (e.g. 20,40)")
637
+
638
+ result = detect(gray, tone)
639
+ if result is None:
640
+ sys.exit(
641
+ "error: no screen-like quad found. The screen's tone probably isn't "
642
+ "separable from its surroundings (a bright UI on a light wall is the "
643
+ "usual case). Try --tone LO,HI with a band read off the screen, or "
644
+ "supply corners by hand to warp.py."
645
+ )
646
+
647
+ corners = result.pop("_corners_np")
648
+ with open(args.out_corners, "w") as f:
649
+ json.dump(result["corners"], f)
650
+ written = {"corners_json": args.out_corners}
651
+
652
+ if args.out_overlay:
653
+ cv2.imwrite(args.out_overlay, draw_overlay(photo, corners), [cv2.IMWRITE_PNG_COMPRESSION, 9])
654
+ written["overlay"] = args.out_overlay
655
+ if args.out_zooms:
656
+ written["zooms"] = write_zooms(photo, corners, args.out_zooms)
657
+
658
+ print(json.dumps({
659
+ **result,
660
+ "photo_size": [photo.shape[1], photo.shape[0]],
661
+ "written": written,
662
+ "advisory": True,
663
+ "accuracy": "First guess, not a measurement. On the reference photo the "
664
+ "worst corner landed ~66px out on a 2400x1792 image — close "
665
+ "enough to aim a human's eye at, not close enough to warp "
666
+ "blind. Auto-detection degrades whenever the screen's tone "
667
+ "runs into a shadowed phone body or a dark background.",
668
+ "next": "Show the overlay and the four zooms to the human and get an "
669
+ "explicit yes. If the quad is off, re-run with --tone LO,HI "
670
+ "(read a band off the screen itself) or hand-correct the "
671
+ "corners. warp.py takes corners explicitly and never calls "
672
+ "this script, so an unconfirmed guess cannot reach a composite.",
673
+ }, indent=1))
674
+
675
+
676
+ if __name__ == "__main__":
677
+ main()