screengraft 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +127 -0
- package/bin/screengraft.js +82 -0
- package/mcp/server.py +322 -0
- package/package.json +59 -0
- package/scripts/detect.py +677 -0
- package/scripts/grade.py +180 -0
- package/scripts/launch.sh +69 -0
- package/scripts/preflight.py +118 -0
- package/scripts/requirements.txt +2 -0
- package/scripts/scan.py +92 -0
- package/scripts/stop.sh +21 -0
- package/scripts/ui.py +503 -0
- package/scripts/warp.py +247 -0
- package/skills/inject-screenshot/SKILL.md +130 -0
- package/ui/fonts/OFL.txt +93 -0
- package/ui/fonts/mona-sans-wordmark.woff2 +0 -0
- package/ui/index.html +1902 -0
|
@@ -0,0 +1,677 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
screengraft — M1a: no-ML screen-quad detection.
|
|
4
|
+
|
|
5
|
+
Finds the four corners of a device screen in a photo, so `warp.py` has a quad
|
|
6
|
+
to warp into. Deterministic and dependency-light: thresholding, morphology and
|
|
7
|
+
line fitting — no model, no download, no network.
|
|
8
|
+
|
|
9
|
+
The pipeline (proven on a real leaning-phone render, 3 Sep 2026):
|
|
10
|
+
|
|
11
|
+
1. Sweep candidate tone bands over the grayscale histogram. A device screen
|
|
12
|
+
is a large region whose pixels sit in a narrow tone band, distinct from
|
|
13
|
+
bezel and background — so instead of guessing one threshold, try every
|
|
14
|
+
band and let the scoring decide.
|
|
15
|
+
2. For each band: close (fills UI text and icons back into the screen body),
|
|
16
|
+
then a large open (kills speckled shadow that would otherwise bridge the
|
|
17
|
+
screen to the background), then take the largest connected component.
|
|
18
|
+
3. Score that blob on how much it looks like a quadrilateral — area, how
|
|
19
|
+
completely it fills its own convex 4-gon, and 4-point approximability.
|
|
20
|
+
Best-scoring band wins.
|
|
21
|
+
4. Fit a line to the straight middle stretch of each of the four edges and
|
|
22
|
+
intersect adjacent lines. This is the step that matters: rounded screen
|
|
23
|
+
corners pull a naive corner estimate inward, and intersecting the straight
|
|
24
|
+
edges recovers the true (virtual) corners the homography needs.
|
|
25
|
+
|
|
26
|
+
Fails honestly. When the screen's tone isn't separable from its surroundings
|
|
27
|
+
(a bright UI photographed against a light wall), no band scores well; the
|
|
28
|
+
script says so and exits non-zero rather than emitting a confident wrong quad.
|
|
29
|
+
That failure is the case SAM 2 (M4) is meant to earn its keep on.
|
|
30
|
+
|
|
31
|
+
Detection never warps anything. It writes corners and an overlay for a human
|
|
32
|
+
to confirm — `warp.py` still requires corners passed explicitly, so an
|
|
33
|
+
unconfirmed guess can never reach the composite.
|
|
34
|
+
|
|
35
|
+
Usage:
|
|
36
|
+
python3 detect.py --photo photo.jpg --out-corners corners.json \
|
|
37
|
+
[--out-overlay overlay.png] [--out-zooms DIR] [--tone LO,HI]
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
import argparse
|
|
41
|
+
import json
|
|
42
|
+
import os
|
|
43
|
+
import sys
|
|
44
|
+
|
|
45
|
+
import cv2
|
|
46
|
+
import numpy as np
|
|
47
|
+
|
|
48
|
+
# Morphology kernel sizes are expressed as a fraction of the image's short
|
|
49
|
+
# side, so behaviour doesn't change with photo resolution.
|
|
50
|
+
CLOSE_FRAC = 0.008 # ~10px on a 1200px-short-side photo: fills UI text
|
|
51
|
+
OPEN_FRAC = 0.023 # ~41px on a 1792px-short-side photo: severs shadow bridges
|
|
52
|
+
EDGE_MIDDLE = 0.70 # fit each edge line on its straight middle 70%
|
|
53
|
+
MIN_AREA_FRAC = 0.01
|
|
54
|
+
MAX_AREA_FRAC = 0.60
|
|
55
|
+
MIN_FILL = 0.60 # the source blob must fill this much of the final quad
|
|
56
|
+
MIN_SIDE_RATIO = 0.08 # reject slivers: shortest side vs longest, after perspective
|
|
57
|
+
# A screen fills MOST of the body it sits in; content drawn on a screen is a
|
|
58
|
+
# small part of it. That one ratio separates "step inward to the screen" from
|
|
59
|
+
# "don't step into a panel", and it arbitrates between the two detectors too.
|
|
60
|
+
NEST_FLOOR = 0.55
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _odd(n: int) -> int:
|
|
64
|
+
n = max(3, int(round(float(n))))
|
|
65
|
+
return n if n % 2 else n + 1
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def blob_for_band(gray: np.ndarray, lo: int, hi: int, close_k: int, open_k: int):
|
|
69
|
+
"""Mask -> morphology -> largest connected component. Returns its contour."""
|
|
70
|
+
mask = cv2.inRange(gray, lo, hi)
|
|
71
|
+
mask = cv2.morphologyEx(
|
|
72
|
+
mask, cv2.MORPH_CLOSE,
|
|
73
|
+
cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (close_k, close_k)),
|
|
74
|
+
)
|
|
75
|
+
mask = cv2.morphologyEx(
|
|
76
|
+
mask, cv2.MORPH_OPEN,
|
|
77
|
+
cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (open_k, open_k)),
|
|
78
|
+
)
|
|
79
|
+
# CHAIN_APPROX_NONE, deliberately: the edge refinement fits a line to each
|
|
80
|
+
# side, and SIMPLE compresses straight runs down to their endpoints, which
|
|
81
|
+
# leaves fitLine with nothing to fit.
|
|
82
|
+
contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
|
|
83
|
+
if not contours:
|
|
84
|
+
return None
|
|
85
|
+
return max(contours, key=cv2.contourArea)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def score_contour(contour, img_area: float):
|
|
89
|
+
"""How much does this blob look like a flat screen? Higher is better."""
|
|
90
|
+
area = cv2.contourArea(contour)
|
|
91
|
+
if area < MIN_AREA_FRAC * img_area or area > MAX_AREA_FRAC * img_area:
|
|
92
|
+
return 0.0, None
|
|
93
|
+
quad = approx_quad(contour)
|
|
94
|
+
if quad is None:
|
|
95
|
+
return 0.0, None
|
|
96
|
+
quad_area = cv2.contourArea(quad.astype(np.float32))
|
|
97
|
+
if quad_area <= 0:
|
|
98
|
+
return 0.0, None
|
|
99
|
+
# Fill ratio: a real screen fills its own quad almost completely. A shadow
|
|
100
|
+
# or a wall patch is ragged and fills far less.
|
|
101
|
+
fill = min(area / quad_area, 1.0)
|
|
102
|
+
return (fill ** 3) * (area / img_area), quad
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# Two "edge support" arbiters — score each candidate quad by how much of its
|
|
106
|
+
# perimeter sits on a Canny edge, then by whether the gradient forms a local
|
|
107
|
+
# ridge across it — were built and measured here on 3 Sep 2026, and both
|
|
108
|
+
# dropped. Numbers, correct answer first:
|
|
109
|
+
#
|
|
110
|
+
# case Canny support ridge support
|
|
111
|
+
# synthetic fixture 0.023 vs 1.000 0.927 vs 0.999 (both prefer wrong)
|
|
112
|
+
# gradient screen -- 0.932 correct (works)
|
|
113
|
+
# real phone photo 0.728 vs 0.339 0.082 vs 0.172 (ridge prefers wrong)
|
|
114
|
+
# real mockup 0.215 correct 0.062 correct (below any usable floor)
|
|
115
|
+
#
|
|
116
|
+
# Edge strength is not a usable arbiter in this domain, for the reason an earlier finding
|
|
117
|
+
# already found from the other direction: a screen's own boundary (glass
|
|
118
|
+
# against a dark bezel) is routinely WEAKER than the device silhouette beside
|
|
119
|
+
# it, and on real photographs both are soft enough that any credibility floor
|
|
120
|
+
# strict enough to reject a wrong quad also rejects the right one. Geometry
|
|
121
|
+
# generalised; photometry did not. detect() arbitrates on nesting instead —
|
|
122
|
+
# the same rule pick_innermost() already uses.
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def score_edge_contour(contour, img_area: float):
|
|
126
|
+
"""Scoring for the Canny path, where score_contour()'s assumptions break.
|
|
127
|
+
|
|
128
|
+
Two differences that matter. Canny traces a closed *ring* around each
|
|
129
|
+
edge, so the contour's own area is an artifact of the ring, not of the
|
|
130
|
+
region — score the QUAD's area instead, which is what gets returned.
|
|
131
|
+
And the fill ratio is correspondingly noisy (it can exceed 1), so it
|
|
132
|
+
weighs in linearly rather than cubed; cubing it once cost the real screen
|
|
133
|
+
the win against a wave-shaped gradient boundary drawn inside that screen
|
|
134
|
+
(fill 0.86 vs a ring artifact's 1.0).
|
|
135
|
+
"""
|
|
136
|
+
quad = approx_quad(contour)
|
|
137
|
+
if quad is None:
|
|
138
|
+
return 0.0, None
|
|
139
|
+
quad_area = float(cv2.contourArea(order_quad(quad).astype(np.float32)))
|
|
140
|
+
if quad_area < MIN_AREA_FRAC * img_area or quad_area > MAX_AREA_FRAC * img_area:
|
|
141
|
+
return 0.0, None
|
|
142
|
+
fill = min(float(cv2.contourArea(contour)) / max(quad_area, 1e-6), 1.0)
|
|
143
|
+
return fill * (quad_area / img_area), quad
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def approx_quad(contour):
|
|
147
|
+
"""Reduce a contour to 4 convex points, or None."""
|
|
148
|
+
hull = cv2.convexHull(contour)
|
|
149
|
+
peri = cv2.arcLength(hull, True)
|
|
150
|
+
for eps in np.arange(0.01, 0.12, 0.005):
|
|
151
|
+
approx = cv2.approxPolyDP(hull, eps * peri, True)
|
|
152
|
+
if len(approx) == 4 and cv2.isContourConvex(approx):
|
|
153
|
+
return approx.reshape(4, 2).astype(np.float64)
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def order_quad(pts: np.ndarray) -> np.ndarray:
|
|
158
|
+
"""Order 4 points TL, TR, BR, BL as they appear in the image."""
|
|
159
|
+
c = pts.mean(axis=0)
|
|
160
|
+
ang = np.arctan2(pts[:, 1] - c[1], pts[:, 0] - c[0])
|
|
161
|
+
pts = pts[np.argsort(ang)] # counter-clockwise in image coords
|
|
162
|
+
start = int(np.argmin(pts.sum(axis=1))) # closest to the image's top-left
|
|
163
|
+
return np.roll(pts, -start, axis=0)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def refine_corners(contour, quad: np.ndarray):
|
|
167
|
+
"""
|
|
168
|
+
Re-derive corners by intersecting fitted edge lines.
|
|
169
|
+
|
|
170
|
+
Rounded screen corners bend the contour inward, so the polygon's own
|
|
171
|
+
vertices sit inside the true corner. Each edge's straight middle stretch,
|
|
172
|
+
extended, meets its neighbour at the corner the homography actually wants.
|
|
173
|
+
|
|
174
|
+
Every contour point is assigned to whichever of the four edges it lies
|
|
175
|
+
nearest, then only the middle stretch of each edge is fitted — which drops
|
|
176
|
+
the rounded corners without needing a distance threshold to tune. The fit
|
|
177
|
+
is Huber rather than least-squares so a bump in the silhouette (a shadow
|
|
178
|
+
nick, a finger) pulls the line far less than it otherwise would.
|
|
179
|
+
|
|
180
|
+
Returns (corners, refined) — `refined` False means it declined and handed
|
|
181
|
+
back the polygon's own corners, which the caller should surface rather
|
|
182
|
+
than quietly present as a refined result.
|
|
183
|
+
"""
|
|
184
|
+
pts = contour.reshape(-1, 2).astype(np.float64)
|
|
185
|
+
if len(pts) < 40:
|
|
186
|
+
return quad, False
|
|
187
|
+
|
|
188
|
+
# Distance from every point to every edge; nearest edge wins the point.
|
|
189
|
+
dists, projections = [], []
|
|
190
|
+
for i in range(4):
|
|
191
|
+
a, b = quad[i], quad[(i + 1) % 4]
|
|
192
|
+
ab = b - a
|
|
193
|
+
length = float(np.linalg.norm(ab))
|
|
194
|
+
if length < 1e-6:
|
|
195
|
+
return quad, False
|
|
196
|
+
u = ab / length
|
|
197
|
+
rel = pts - a
|
|
198
|
+
dists.append(np.abs(u[0] * rel[:, 1] - u[1] * rel[:, 0]))
|
|
199
|
+
projections.append((rel @ ab) / (length ** 2))
|
|
200
|
+
nearest = np.argmin(np.vstack(dists), axis=0)
|
|
201
|
+
|
|
202
|
+
margin = (1.0 - EDGE_MIDDLE) / 2.0
|
|
203
|
+
lines = []
|
|
204
|
+
for i in range(4):
|
|
205
|
+
t = projections[i]
|
|
206
|
+
sel = pts[(nearest == i) & (t > margin) & (t < 1.0 - margin)]
|
|
207
|
+
if len(sel) < 10:
|
|
208
|
+
return quad, False
|
|
209
|
+
vx, vy, x0, y0 = cv2.fitLine(
|
|
210
|
+
sel.astype(np.float32), cv2.DIST_HUBER, 0, 0.01, 0.01
|
|
211
|
+
).ravel()
|
|
212
|
+
lines.append((float(vx), float(vy), float(x0), float(y0)))
|
|
213
|
+
|
|
214
|
+
corners = []
|
|
215
|
+
for i in range(4):
|
|
216
|
+
# Corner i is where edge (i-1) meets edge i.
|
|
217
|
+
vx1, vy1, x1, y1 = lines[(i - 1) % 4]
|
|
218
|
+
vx2, vy2, x2, y2 = lines[i]
|
|
219
|
+
denom = vx1 * vy2 - vy1 * vx2
|
|
220
|
+
if abs(denom) < 1e-9:
|
|
221
|
+
return quad, False # parallel edges: refinement is meaningless
|
|
222
|
+
t = ((x2 - x1) * vy2 - (y2 - y1) * vx2) / denom
|
|
223
|
+
corners.append([x1 + t * vx1, y1 + t * vy1])
|
|
224
|
+
corners = np.array(corners, dtype=np.float64)
|
|
225
|
+
|
|
226
|
+
# A refinement that moves a corner further than a quarter of the quad's
|
|
227
|
+
# size is not a refinement; something upstream was wrong.
|
|
228
|
+
span = float(np.linalg.norm(quad[2] - quad[0]))
|
|
229
|
+
if np.max(np.linalg.norm(corners - quad, axis=1)) > 0.25 * span:
|
|
230
|
+
return quad, False
|
|
231
|
+
return corners, True
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def draw_overlay(photo: np.ndarray, corners: np.ndarray) -> np.ndarray:
|
|
235
|
+
out = photo.copy()
|
|
236
|
+
poly = corners.astype(np.int32)
|
|
237
|
+
cv2.polylines(out, [poly], True, (0, 255, 0), 2, cv2.LINE_AA)
|
|
238
|
+
for (x, y), label in zip(corners, ["TL", "TR", "BR", "BL"], strict=True):
|
|
239
|
+
p = (int(round(x)), int(round(y)))
|
|
240
|
+
cv2.circle(out, p, 9, (0, 0, 255), 2, cv2.LINE_AA)
|
|
241
|
+
cv2.circle(out, p, 2, (0, 0, 255), -1, cv2.LINE_AA)
|
|
242
|
+
cv2.putText(out, label, (p[0] + 12, p[1] - 12),
|
|
243
|
+
cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2, cv2.LINE_AA)
|
|
244
|
+
return out
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def write_zooms(photo: np.ndarray, corners: np.ndarray, out_dir: str, box: int = 140):
|
|
248
|
+
"""Corner close-ups — what a human needs to actually confirm a quad."""
|
|
249
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
250
|
+
h, w = photo.shape[:2]
|
|
251
|
+
paths = []
|
|
252
|
+
overlaid = draw_overlay(photo, corners)
|
|
253
|
+
for (x, y), label in zip(corners, ["TL", "TR", "BR", "BL"], strict=True):
|
|
254
|
+
cx, cy = int(round(x)), int(round(y))
|
|
255
|
+
x0, y0 = max(0, cx - box), max(0, cy - box)
|
|
256
|
+
x1, y1 = min(w, cx + box), min(h, cy + box)
|
|
257
|
+
crop = overlaid[y0:y1, x0:x1]
|
|
258
|
+
if crop.size == 0:
|
|
259
|
+
continue
|
|
260
|
+
crop = cv2.resize(crop, None, fx=2, fy=2, interpolation=cv2.INTER_NEAREST)
|
|
261
|
+
path = os.path.join(out_dir, f"zoom_{label}.png")
|
|
262
|
+
cv2.imwrite(path, crop, [cv2.IMWRITE_PNG_COMPRESSION, 9])
|
|
263
|
+
paths.append(path)
|
|
264
|
+
return paths
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def quad_contains(outer: np.ndarray, inner: np.ndarray, margin: float = 2.0) -> bool:
|
|
268
|
+
"""Is `inner` wholly inside `outer`?"""
|
|
269
|
+
poly = outer.astype(np.float32)
|
|
270
|
+
return all(
|
|
271
|
+
cv2.pointPolygonTest(poly, (float(x), float(y)), True) > margin
|
|
272
|
+
for x, y in inner
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def overlap_frac(inner: np.ndarray, outer: np.ndarray) -> float:
|
|
277
|
+
"""How much of `inner`'s area falls inside `outer` (0..1)."""
|
|
278
|
+
a = float(cv2.contourArea(inner.astype(np.float32)))
|
|
279
|
+
if a <= 0:
|
|
280
|
+
return 0.0
|
|
281
|
+
# The first return is the intersection area, deliberately discarded: it is
|
|
282
|
+
# recomputed from `region` below so this function has exactly one notion of
|
|
283
|
+
# area, measured the same way as `a`.
|
|
284
|
+
_, region = cv2.intersectConvexConvex(inner.astype(np.float32),
|
|
285
|
+
outer.astype(np.float32))
|
|
286
|
+
if region is None or len(region) < 3:
|
|
287
|
+
return 0.0
|
|
288
|
+
return float(cv2.contourArea(region.astype(np.float32))) / a
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def pick_innermost(candidates, best):
|
|
292
|
+
"""
|
|
293
|
+
A device photo offers more than one screen-shaped region: the glass screen,
|
|
294
|
+
and the bezel or body it sits in. They nest, and the outer one wins on
|
|
295
|
+
area alone — so once a best candidate is found, step inward while a
|
|
296
|
+
comparably screen-like quad sits wholly inside it. The innermost such
|
|
297
|
+
region is the screen; that's the one the UI has to be warped into.
|
|
298
|
+
|
|
299
|
+
The nesting floor is deliberately high. A screen fills most of the body
|
|
300
|
+
it sits in, so a genuine step inward barely shrinks. Content *drawn on*
|
|
301
|
+
the screen — a panel, a card, a gradient band — is much smaller relative
|
|
302
|
+
to its parent, and stepping into it is a miss, not a refinement. At the
|
|
303
|
+
old 0.35 floor a wave-shaped gradient at 49% of its screen was eligible;
|
|
304
|
+
at 0.55 it isn't (an earlier finding, 3 Sep 2026).
|
|
305
|
+
"""
|
|
306
|
+
current = best
|
|
307
|
+
for _ in range(4): # screen inside bezel inside body: a few steps is plenty
|
|
308
|
+
c_area = cv2.contourArea(current[1].astype(np.float32))
|
|
309
|
+
inner = [
|
|
310
|
+
c for c in candidates
|
|
311
|
+
if c is not current
|
|
312
|
+
and c[0] >= 0.25 * current[0]
|
|
313
|
+
and quad_contains(current[1], c[1])
|
|
314
|
+
and NEST_FLOOR * c_area <= cv2.contourArea(c[1].astype(np.float32)) < c_area
|
|
315
|
+
]
|
|
316
|
+
if not inner:
|
|
317
|
+
return current
|
|
318
|
+
# Largest of the nested ones: the screen, not a panel drawn on it.
|
|
319
|
+
current = max(inner, key=lambda c: cv2.contourArea(c[1].astype(np.float32)))
|
|
320
|
+
return current
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def measure_corner_radius(contour, corners: np.ndarray):
|
|
324
|
+
"""Estimate the screen's corner radius from the mask outline.
|
|
325
|
+
|
|
326
|
+
For a rounded corner of radius r, the outline's nearest point to the
|
|
327
|
+
virtual (sharp) corner lies on the bisector at distance r*(sqrt(2)-1).
|
|
328
|
+
So r ~= d / 0.4142 per corner. Perspective skews this a little, so we
|
|
329
|
+
report the median over the four corners, express it as a fraction of
|
|
330
|
+
the screen's width (so warp.py can turn it into screenshot pixels), and
|
|
331
|
+
flag confidence from how well the four corners agree.
|
|
332
|
+
"""
|
|
333
|
+
pts = contour.reshape(-1, 2).astype(np.float64)
|
|
334
|
+
top = np.linalg.norm(corners[1] - corners[0])
|
|
335
|
+
bottom = np.linalg.norm(corners[2] - corners[3])
|
|
336
|
+
width = float((top + bottom) / 2.0)
|
|
337
|
+
per = []
|
|
338
|
+
for c in corners:
|
|
339
|
+
d = float(np.min(np.linalg.norm(pts - c, axis=1)))
|
|
340
|
+
per.append(d / (np.sqrt(2.0) - 1.0))
|
|
341
|
+
per = np.array(per)
|
|
342
|
+
r = float(np.median(per))
|
|
343
|
+
spread = float((per.max() - per.min()) / max(r, 1e-6))
|
|
344
|
+
confident = bool(r > 2.0 and spread < 0.5)
|
|
345
|
+
return {
|
|
346
|
+
"photo_px": round(r, 1),
|
|
347
|
+
"frac_of_width": round(r / width, 4) if width > 0 else 0.0,
|
|
348
|
+
"per_corner_px": [round(float(v), 1) for v in per],
|
|
349
|
+
"confident": confident,
|
|
350
|
+
"note": ("Median of four per-corner estimates; spread %.0f%%." % (spread * 100))
|
|
351
|
+
+ (" Good agreement." if confident else
|
|
352
|
+
" Poor agreement or near-square corners: offer device presets instead."),
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
# A local, per-handle version of refine_corners() below — "snap this one
|
|
357
|
+
# corner to its nearest edge on release" — was prototyped and measured
|
|
358
|
+
# against the real reference photo (see build-brief.md, "an earlier finding" section): mean
|
|
359
|
+
# error did not improve over the rough dragged position (19.6px unrefined vs
|
|
360
|
+
# 21.5px refined), because the true bezel/screen edge is often LOWER local
|
|
361
|
+
# contrast than a nearby wrong edge (e.g. bezel against a bright background),
|
|
362
|
+
# so "strongest/nearest gradient" reliably locks onto the wrong one. Dropped
|
|
363
|
+
# rather than wired into the UI. detect()'s global refine_corners() below is
|
|
364
|
+
# a different, more reliable case — it fits all four edges from one already-
|
|
365
|
+
# segmented contour, so it isn't picking between competing nearby edges.
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def validate_quad(corners: np.ndarray, contour, img_area: float):
|
|
369
|
+
"""Is this *final* quad plausibly a screen? Returns (ok, reason).
|
|
370
|
+
|
|
371
|
+
The area gate in score_contour() runs against the CONTOUR, before
|
|
372
|
+
approxPolyDP and refine_corners() reshape it — so a contour that scrapes
|
|
373
|
+
past the floor can still yield a final quad far below it. That is exactly
|
|
374
|
+
how a 0.72%-of-image sliver around a watermark was once returned as a
|
|
375
|
+
confident detection (3 Sep 2026, an earlier finding). Everything returned to a caller
|
|
376
|
+
goes through here.
|
|
377
|
+
"""
|
|
378
|
+
area = float(cv2.contourArea(corners.astype(np.float32)))
|
|
379
|
+
frac = area / img_area
|
|
380
|
+
if frac < MIN_AREA_FRAC:
|
|
381
|
+
return False, ("final quad covers only %.2f%% of the photo (floor %.0f%%) "
|
|
382
|
+
"— too small to be the screen" % (frac * 100, MIN_AREA_FRAC * 100))
|
|
383
|
+
if frac > MAX_AREA_FRAC:
|
|
384
|
+
return False, ("final quad covers %.0f%% of the photo (ceiling %.0f%%) "
|
|
385
|
+
"— that's the scene, not a screen" % (frac * 100, MAX_AREA_FRAC * 100))
|
|
386
|
+
if not cv2.isContourConvex(corners.astype(np.float32).reshape(-1, 1, 2)):
|
|
387
|
+
return False, "final quad is not convex — a plane in perspective always is"
|
|
388
|
+
sides = [float(np.linalg.norm(corners[(i + 1) % 4] - corners[i])) for i in range(4)]
|
|
389
|
+
if min(sides) < MIN_SIDE_RATIO * max(sides):
|
|
390
|
+
return False, ("final quad is a sliver (shortest side %.0f%% of the longest) "
|
|
391
|
+
% (100 * min(sides) / max(sides)))
|
|
392
|
+
if contour is not None:
|
|
393
|
+
fill = float(cv2.contourArea(contour)) / max(area, 1e-6)
|
|
394
|
+
if fill < MIN_FILL:
|
|
395
|
+
return False, ("source blob fills only %.0f%% of the final quad "
|
|
396
|
+
"(floor %.0f%%) — ragged, not a screen"
|
|
397
|
+
% (fill * 100, MIN_FILL * 100))
|
|
398
|
+
return True, ""
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _finalize(candidates, img_area: float, refine: bool = True):
|
|
402
|
+
"""Best candidate that survives refinement AND validation.
|
|
403
|
+
|
|
404
|
+
Walks candidates best-score-first rather than trusting the top one: a
|
|
405
|
+
high-scoring blob whose refined quad fails the plausibility gate is a
|
|
406
|
+
miss, not a result, and the next candidate deserves a look before the
|
|
407
|
+
detector gives up.
|
|
408
|
+
|
|
409
|
+
`refine` is off for the Canny path. refine_corners() assumes the contour
|
|
410
|
+
is a filled region's silhouette, where each side has one long straight
|
|
411
|
+
run to fit. A Canny contour is a ring tracing both sides of an edge, with
|
|
412
|
+
content edges caught inside it, so the per-edge line fits pick up the
|
|
413
|
+
wrong points: measured on the gradient-screen mockup it pushed one corner
|
|
414
|
+
78px off an otherwise correct quad . The polygon approximation of a
|
|
415
|
+
Canny boundary is already on the edge, so there is nothing to recover.
|
|
416
|
+
"""
|
|
417
|
+
rejected = []
|
|
418
|
+
for cand in sorted(candidates, key=lambda c: c[0], reverse=True):
|
|
419
|
+
score, quad, contour, tag = pick_innermost(candidates, cand)
|
|
420
|
+
if refine:
|
|
421
|
+
refined, did_refine = refine_corners(contour, quad)
|
|
422
|
+
else:
|
|
423
|
+
refined, did_refine = quad, False
|
|
424
|
+
corners = order_quad(refined)
|
|
425
|
+
ok, why = validate_quad(corners, contour, img_area)
|
|
426
|
+
if ok:
|
|
427
|
+
return {
|
|
428
|
+
"corners": [[round(float(x), 1), round(float(y), 1)] for x, y in corners],
|
|
429
|
+
"score": round(float(score), 5),
|
|
430
|
+
"edge_refined": did_refine,
|
|
431
|
+
"corner_radius": measure_corner_radius(contour, corners),
|
|
432
|
+
"_corners_np": corners,
|
|
433
|
+
"_tag": tag,
|
|
434
|
+
"rejected": rejected,
|
|
435
|
+
}
|
|
436
|
+
rejected.append({"score": round(float(score), 5), "tag": tag, "why": why})
|
|
437
|
+
return None
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def detect_tone(gray: np.ndarray, tone=None):
|
|
441
|
+
"""Tone-band segmentation. Assumes the screen sits in a narrow tone band."""
|
|
442
|
+
h, w = gray.shape[:2]
|
|
443
|
+
img_area = float(h * w)
|
|
444
|
+
short = min(h, w)
|
|
445
|
+
close_k = _odd(CLOSE_FRAC * short)
|
|
446
|
+
open_k = _odd(OPEN_FRAC * short)
|
|
447
|
+
|
|
448
|
+
if tone is not None:
|
|
449
|
+
bands = [tone]
|
|
450
|
+
else:
|
|
451
|
+
# 32 bins over the 0-255 range; try each bin, and each adjacent pair
|
|
452
|
+
# (a screen showing a gradient can straddle two bins).
|
|
453
|
+
edges = [int(round(i * 256 / 32)) for i in range(33)]
|
|
454
|
+
bands = [(edges[i], edges[i + 1] - 1) for i in range(32)]
|
|
455
|
+
bands += [(edges[i], edges[i + 2] - 1) for i in range(31)]
|
|
456
|
+
|
|
457
|
+
candidates = []
|
|
458
|
+
for lo, hi in bands:
|
|
459
|
+
contour = blob_for_band(gray, lo, hi, close_k, open_k)
|
|
460
|
+
if contour is None:
|
|
461
|
+
continue
|
|
462
|
+
score, quad = score_contour(contour, img_area)
|
|
463
|
+
if score > 0 and quad is not None:
|
|
464
|
+
# Prefer a narrow band. A screen is tonally uniform; a band wide
|
|
465
|
+
# enough to also swallow the phone's shadowed body scores well on
|
|
466
|
+
# area but produces edges that follow the body, not the glass.
|
|
467
|
+
# Measured on the reference photo: this alone cut the worst-corner
|
|
468
|
+
# error from 110px to 66px.
|
|
469
|
+
score *= (16.0 / (hi - lo + 1)) ** 0.25
|
|
470
|
+
candidates.append((score, order_quad(quad), contour, (lo, hi)))
|
|
471
|
+
|
|
472
|
+
res = _finalize(candidates, img_area)
|
|
473
|
+
if res is None:
|
|
474
|
+
return None
|
|
475
|
+
band = res.pop("_tag")
|
|
476
|
+
res["method"] = "tone"
|
|
477
|
+
res["band"] = [int(band[0]), int(band[1])]
|
|
478
|
+
return res
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def detect_edges(gray: np.ndarray):
|
|
482
|
+
"""Canny-and-quad detection — the document-scanner path.
|
|
483
|
+
|
|
484
|
+
Tone banding assumes a near-uniform screen, which breaks the moment the
|
|
485
|
+
screen shows real UI: a gradient wallpaper never lands inside one band,
|
|
486
|
+
so the sweep finds nothing (or worse, something else — an earlier finding, 3 Sep 2026).
|
|
487
|
+
This keys on the *boundary* instead of the fill, so screen content is
|
|
488
|
+
irrelevant; all it needs is contrast between glass and bezel, which a
|
|
489
|
+
device photo has by construction.
|
|
490
|
+
"""
|
|
491
|
+
h, w = gray.shape[:2]
|
|
492
|
+
img_area = float(h * w)
|
|
493
|
+
short = min(h, w)
|
|
494
|
+
blur = cv2.GaussianBlur(gray, (5, 5), 0)
|
|
495
|
+
|
|
496
|
+
candidates = []
|
|
497
|
+
# Sweep the Canny thresholds off the image's own median rather than fixed
|
|
498
|
+
# numbers, then a few sigmas around it — one exposure doesn't suit both a
|
|
499
|
+
# bright render and a dim photo.
|
|
500
|
+
med = float(np.median(blur))
|
|
501
|
+
for sigma in (0.20, 0.33, 0.50, 0.66):
|
|
502
|
+
lo = int(max(0, (1.0 - sigma) * med))
|
|
503
|
+
hi = int(min(255, (1.0 + sigma) * med))
|
|
504
|
+
if hi <= lo:
|
|
505
|
+
continue
|
|
506
|
+
edges = cv2.Canny(blur, lo, hi, L2gradient=True)
|
|
507
|
+
# Close small gaps so a bezel outline broken by a notch or a glare
|
|
508
|
+
# spot still forms one closed contour.
|
|
509
|
+
k = _odd(0.004 * short)
|
|
510
|
+
edges = cv2.morphologyEx(
|
|
511
|
+
edges, cv2.MORPH_CLOSE,
|
|
512
|
+
cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k)),
|
|
513
|
+
)
|
|
514
|
+
contours, _ = cv2.findContours(edges, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
|
|
515
|
+
for c in contours:
|
|
516
|
+
score, quad = score_edge_contour(c, img_area)
|
|
517
|
+
if score > 0 and quad is not None:
|
|
518
|
+
candidates.append((score, order_quad(quad), c, (lo, hi)))
|
|
519
|
+
|
|
520
|
+
res = _finalize(candidates, img_area, refine=False)
|
|
521
|
+
if res is None:
|
|
522
|
+
return None
|
|
523
|
+
thr = res.pop("_tag")
|
|
524
|
+
res["method"] = "edge"
|
|
525
|
+
res["canny"] = [int(thr[0]), int(thr[1])]
|
|
526
|
+
return res
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def detect(gray: np.ndarray, tone=None, method="auto"):
|
|
530
|
+
"""Run both detectors; arbitrate on how the two quads nest.
|
|
531
|
+
|
|
532
|
+
The two fail on opposite things. Tone banding needs a tonally uniform
|
|
533
|
+
screen: it finds nothing when the screen shows real UI, or — worse — a
|
|
534
|
+
horizontal slab of a gradient wallpaper, which looks like a perfectly
|
|
535
|
+
good screen. Edge detection needs only a visible boundary, so it handles
|
|
536
|
+
content-filled screens, but with nothing to tell glass from bezel it will
|
|
537
|
+
return the device's outer silhouette instead.
|
|
538
|
+
|
|
539
|
+
Ranking by score is meaningless (the two are computed differently), and
|
|
540
|
+
two photometric arbiters were built, measured and rejected — see the note
|
|
541
|
+
above score_edge_contour(). What does separate the cases is the same
|
|
542
|
+
geometry pick_innermost() already relies on: a screen fills MOST of the
|
|
543
|
+
body it sits in, while content drawn on a screen is a small part of it.
|
|
544
|
+
So when tone's quad sits inside edge's, the area ratio says which is
|
|
545
|
+
which — 0.79 on the fixture (screen in body: take the inner, tone), 0.41
|
|
546
|
+
on a gradient screen (slab on screen: take the outer, edge). When they
|
|
547
|
+
don't nest at all, neither is a subregion of the other and tone wins,
|
|
548
|
+
since its assumptions being met is itself evidence and it refines corners.
|
|
549
|
+
|
|
550
|
+
Disagreement is reported, never silently resolved — the human confirms
|
|
551
|
+
the corners either way.
|
|
552
|
+
"""
|
|
553
|
+
results = []
|
|
554
|
+
if method in ("auto", "tone"):
|
|
555
|
+
r = detect_tone(gray, tone)
|
|
556
|
+
if r:
|
|
557
|
+
results.append(r)
|
|
558
|
+
if method in ("auto", "edge") and tone is None:
|
|
559
|
+
r = detect_edges(gray)
|
|
560
|
+
if r:
|
|
561
|
+
results.append(r)
|
|
562
|
+
|
|
563
|
+
if not results:
|
|
564
|
+
return None
|
|
565
|
+
|
|
566
|
+
t = next((r for r in results if r["method"] == "tone"), None)
|
|
567
|
+
e = next((r for r in results if r["method"] == "edge"), None)
|
|
568
|
+
why = ""
|
|
569
|
+
if t and e:
|
|
570
|
+
tq, eq = t["_corners_np"], e["_corners_np"]
|
|
571
|
+
ta = float(cv2.contourArea(tq.astype(np.float32)))
|
|
572
|
+
ea = float(cv2.contourArea(eq.astype(np.float32)))
|
|
573
|
+
nested = overlap_frac(tq, eq) >= 0.90 and ta < ea
|
|
574
|
+
ratio = ta / ea if ea > 0 else 0.0
|
|
575
|
+
if nested and ratio < NEST_FLOOR:
|
|
576
|
+
best, why = e, ("the tone quad is only %.0f%% of the edge quad it sits "
|
|
577
|
+
"inside — that's content drawn on the screen, not the "
|
|
578
|
+
"screen" % (ratio * 100))
|
|
579
|
+
elif nested:
|
|
580
|
+
best, why = t, ("the tone quad sits inside the edge quad at %.0f%% of "
|
|
581
|
+
"its area — a screen inside a device body" % (ratio * 100))
|
|
582
|
+
else:
|
|
583
|
+
best, why = t, "the two quads aren't nested; the tone detector refines corners"
|
|
584
|
+
else:
|
|
585
|
+
best = t or e
|
|
586
|
+
|
|
587
|
+
if len(results) == 2:
|
|
588
|
+
a, b = (r["_corners_np"] for r in results)
|
|
589
|
+
diag = float(np.hypot(*gray.shape[:2]))
|
|
590
|
+
spread = float(np.max(np.linalg.norm(a - b, axis=1))) / diag
|
|
591
|
+
agree = bool(spread < 0.02)
|
|
592
|
+
best["agreement"] = {
|
|
593
|
+
"both_found": True,
|
|
594
|
+
"max_corner_gap_frac_of_diagonal": round(spread, 4),
|
|
595
|
+
"agree": agree,
|
|
596
|
+
"note": ("Both detectors independently landed on the same quad — "
|
|
597
|
+
"that is real evidence, not one algorithm's opinion."
|
|
598
|
+
if agree else
|
|
599
|
+
"The two detectors disagree by %.0f%% of the image diagonal. "
|
|
600
|
+
"Showing the %s one because %s — but check all four corners."
|
|
601
|
+
% (spread * 100, best["method"], why)),
|
|
602
|
+
}
|
|
603
|
+
best["agreement"]["chosen_because"] = why
|
|
604
|
+
other = next(r for r in results if r is not best)
|
|
605
|
+
best["other"] = {"method": other["method"], "corners": other["corners"]}
|
|
606
|
+
else:
|
|
607
|
+
best["agreement"] = {
|
|
608
|
+
"both_found": False,
|
|
609
|
+
"agree": False,
|
|
610
|
+
"note": ("Only the %s detector found anything — a first guess to "
|
|
611
|
+
"correct, not a measurement." % best["method"]),
|
|
612
|
+
}
|
|
613
|
+
return best
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def main() -> None:
|
|
617
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
618
|
+
ap.add_argument("--photo", required=True)
|
|
619
|
+
ap.add_argument("--out-corners", required=True, help="Where to write the corners JSON")
|
|
620
|
+
ap.add_argument("--out-overlay", help="Optional PNG showing the detected quad on the photo")
|
|
621
|
+
ap.add_argument("--out-zooms", help="Optional directory for 2x corner close-ups")
|
|
622
|
+
ap.add_argument("--tone", help="Skip the sweep and force one band, e.g. --tone 20,40")
|
|
623
|
+
args = ap.parse_args()
|
|
624
|
+
|
|
625
|
+
photo = cv2.imread(args.photo, cv2.IMREAD_COLOR)
|
|
626
|
+
if photo is None:
|
|
627
|
+
sys.exit(f"error: could not read photo: {args.photo}")
|
|
628
|
+
gray = cv2.cvtColor(photo, cv2.COLOR_BGR2GRAY)
|
|
629
|
+
|
|
630
|
+
tone = None
|
|
631
|
+
if args.tone:
|
|
632
|
+
try:
|
|
633
|
+
lo, hi = (int(v) for v in args.tone.split(","))
|
|
634
|
+
tone = (lo, hi)
|
|
635
|
+
except ValueError:
|
|
636
|
+
sys.exit("error: --tone must be LO,HI (e.g. 20,40)")
|
|
637
|
+
|
|
638
|
+
result = detect(gray, tone)
|
|
639
|
+
if result is None:
|
|
640
|
+
sys.exit(
|
|
641
|
+
"error: no screen-like quad found. The screen's tone probably isn't "
|
|
642
|
+
"separable from its surroundings (a bright UI on a light wall is the "
|
|
643
|
+
"usual case). Try --tone LO,HI with a band read off the screen, or "
|
|
644
|
+
"supply corners by hand to warp.py."
|
|
645
|
+
)
|
|
646
|
+
|
|
647
|
+
corners = result.pop("_corners_np")
|
|
648
|
+
with open(args.out_corners, "w") as f:
|
|
649
|
+
json.dump(result["corners"], f)
|
|
650
|
+
written = {"corners_json": args.out_corners}
|
|
651
|
+
|
|
652
|
+
if args.out_overlay:
|
|
653
|
+
cv2.imwrite(args.out_overlay, draw_overlay(photo, corners), [cv2.IMWRITE_PNG_COMPRESSION, 9])
|
|
654
|
+
written["overlay"] = args.out_overlay
|
|
655
|
+
if args.out_zooms:
|
|
656
|
+
written["zooms"] = write_zooms(photo, corners, args.out_zooms)
|
|
657
|
+
|
|
658
|
+
print(json.dumps({
|
|
659
|
+
**result,
|
|
660
|
+
"photo_size": [photo.shape[1], photo.shape[0]],
|
|
661
|
+
"written": written,
|
|
662
|
+
"advisory": True,
|
|
663
|
+
"accuracy": "First guess, not a measurement. On the reference photo the "
|
|
664
|
+
"worst corner landed ~66px out on a 2400x1792 image — close "
|
|
665
|
+
"enough to aim a human's eye at, not close enough to warp "
|
|
666
|
+
"blind. Auto-detection degrades whenever the screen's tone "
|
|
667
|
+
"runs into a shadowed phone body or a dark background.",
|
|
668
|
+
"next": "Show the overlay and the four zooms to the human and get an "
|
|
669
|
+
"explicit yes. If the quad is off, re-run with --tone LO,HI "
|
|
670
|
+
"(read a band off the screen itself) or hand-correct the "
|
|
671
|
+
"corners. warp.py takes corners explicitly and never calls "
|
|
672
|
+
"this script, so an unconfirmed guess cannot reach a composite.",
|
|
673
|
+
}, indent=1))
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
if __name__ == "__main__":
|
|
677
|
+
main()
|