screengraft 0.38.0 → 0.40.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screengraft",
-  "version": "0.38.0",
+  "version": "0.40.0",
   "description": "Put a UI screenshot or screen recording onto a photographed device screen with the perspective exactly right \u2014 a homography you confirm by hand, not a generative guess.",
   "keywords": [
     "mockup",

package/scripts/detect.py

@@ -797,12 +797,31 @@ def detect_edges(gray: np.ndarray, click=None, trace=None):
     # Sweep the Canny thresholds off the image's own median rather than fixed
     # numbers, then a few sigmas around it — one exposure doesn't suit both a
     # bright render and a dim photo.
+    #
+    # ... and ALSO off Otsu's threshold, because the median anchor has a hole
+    # the median cannot see: a dark photograph. A black phone on a black
+    # backdrop has a median of 0..4, so every sweep point lands at hi <= 7 and
+    # Canny fires on every pixel of noise -- the edge map is a solid sheet and
+    # no closed quad survives it. Both photographs in the labelled corpus on
+    # which nothing near the screen was EVER proposed (11 Sep 2026) are exactly
+    # this, and edge returned zero candidates on them. Otsu splits the two
+    # modes that are actually there -- 97 and 127 on those two -- and the same
+    # sweep then proposes the screen at 9% and 12% unrefined, which is where
+    # every other photograph's nearest candidate sits. On the other seven the
+    # nearest candidate is unchanged. The saturation channel already anchors
+    # on Otsu for the same reason (one percentile fails when the thing sought
+    # is smaller than the percentile).
     med = float(np.median(blur))
-    for sigma in (0.20, 0.33, 0.50, 0.66):
-        lo = int(max(0, (1.0 - sigma) * med))
-        hi = int(min(255, (1.0 + sigma) * med))
-        if hi <= lo:
+    otsu, _ = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+    sweep = [((1.0 - s) * med, (1.0 + s) * med) for s in (0.20, 0.33, 0.50, 0.66)]
+    sweep += [(otsu * f / 2.0, otsu * f) for f in (0.5, 1.0, 1.5)]
+    seen = set()
+    for lo_f, hi_f in sweep:
+        lo = int(max(0, lo_f))
+        hi = int(min(255, hi_f))
+        if hi <= lo or (lo, hi) in seen:
             continue
+        seen.add((lo, hi))
         edges = cv2.Canny(blur, lo, hi, L2gradient=True)
         # Close small gaps so a bezel outline broken by a notch or a glare
         # spot still forms one closed contour.
@@ -996,22 +1015,63 @@ def arbitrate(results, shape, click=None):
     sat = next((r for r in results if r["method"] == "saturation"), None)
     why = "it was the only detector left after the rounded-corner filter" \
         if len(results) == 1 else "its tone band assumption held, which is itself evidence"
-    if t and e:
-        tq, eq = t["_corners_np"], e["_corners_np"]
-        ta = float(cv2.contourArea(tq.astype(np.float32)))
+    # ONE region-vs-edge arbitration, for whichever region channel survived the
+    # rounded-corner filter above. Until 11 Sep 2026 only tone got the nesting
+    # rules; saturation-vs-edge fell through to "saturation wins". On iPhone-4
+    # that returned a saturation quad 18% off, nested around an edge quad 3%
+    # off at 94% of its area -- exactly the "screen inside a body" shape the
+    # tone branch already knew how to read.
+    region = t if t is not None else sat
+    rname = "tone" if t is not None else "saturation"
+    if region is not None and e is not None:
+        rq, eq = region["_corners_np"], e["_corners_np"]
+        ra = float(cv2.contourArea(rq.astype(np.float32)))
         ea = float(cv2.contourArea(eq.astype(np.float32)))
-        nested = overlap_frac(tq, eq) >= 0.90 and ta < ea
-        ratio = ta / ea if ea > 0 else 0.0
-        if nested and ratio < NEST_FLOOR:
-            best, why = e, ("the tone quad is only %.0f%% of the edge quad it sits "
-                            "inside — that's content drawn on the screen, not the "
-                            "screen" % (ratio * 100))
+        # Nesting is read on whichever quad is inside the other. It used to be
+        # read only with the region quad inside the edge quad, so an edge quad
+        # sitting inside a region quad (iPhone-4: edge 3% off inside a
+        # saturation quad 18% off at 94%) was "not nested" and fell through to
+        # the region winning by default.
+        inner, outer = (region, e) if ra < ea else (e, region)
+        iname, oname = (rname, "edge") if ra < ea else ("edge", rname)
+        nested = overlap_frac(inner["_corners_np"], outer["_corners_np"]) >= 0.90
+        ratio = min(ra, ea) / max(ra, ea) if max(ra, ea) > 0 else 0.0
+        ti, to = shape_tier(inner), shape_tier(outer)
+        tr, te = shape_tier(region), shape_tier(e)
+        # Nested: shape evidence first, the area ratio only to break a tie.
+        #
+        # The ratio alone reads "small inside big" as content-inside-screen and
+        # "large inside big" as screen-inside-body, and it is right on the
+        # gradient-screen fixture (41%: a slab on the screen) and on most
+        # photographs. It is wrong exactly when the quads' own corners say the
+        # opposite: a UI content region at 96% of the screen with loose corners
+        # (two of ten labelled photographs, 15% off) is not a screen inside a
+        # body, and a confident phone screen at 34% of a loosely-rounded table
+        # (the hand-built case in test_detect) is not content on it. Tiers
+        # decide those; at equal tiers the ratio still does, unchanged.
+        if nested and to > ti:
+            best, why = outer, ("the %s quad sits inside the %s quad at %.0f%% of its "
+                                "area, but the outer quad's corners agree on a radius "
+                                "(tier %d) and the inner's do not (tier %d) — content "
+                                "inside a screen, not a screen inside a body"
+                                % (iname, oname, ratio * 100, to, ti))
+        elif nested and ti > to:
+            best, why = inner, ("the %s quad sits inside the %s quad at %.0f%% of its "
+                                "area, and the inner quad's corners agree on a radius "
+                                "(tier %d) where the outer's do not (tier %d) — a "
+                                "screen on something larger"
+                                % (iname, oname, ratio * 100, ti, to))
+        elif nested and ratio < NEST_FLOOR:
+            best, why = outer, ("the %s quad is only %.0f%% of the %s quad it sits "
+                                "inside — that's content drawn on the screen, not the "
+                                "screen" % (iname, ratio * 100, oname))
         elif nested:
-            best, why = t, ("the tone quad sits inside the edge quad at %.0f%% of "
-                            "its area — a screen inside a device body" % (ratio * 100))
-        elif shape_tier(e) > shape_tier(t):
+            best, why = inner, ("the %s quad sits inside the %s quad at %.0f%% of "
+                                "its area — a screen inside a device body"
+                                % (iname, oname, ratio * 100))
+        elif te > tr:
             # Not nested, and the edge quad's own shape says "screen" more
-            # strongly than tone's does. Before 11 Sep 2026 tone won here
+            # strongly than the region's does. Before 11 Sep 2026 tone won here
             # unconditionally, on the argument that its band assumption holding
             # was itself evidence -- and on two of ten labelled photographs that
             # handed the answer to a tone quad 159% and 249% off, with corner
@@ -1019,22 +1079,12 @@ def arbitrate(results, shape, click=None):
             # radius. A band assumption is weaker evidence than four agreeing
             # corners; the tiers say so and this reads them.
             best, why = e, ("the two quads aren't nested and the edge quad's "
-                            "corners agree on a radius (tier %d) where tone's do "
-                            "not (tier %d)" % (shape_tier(e), shape_tier(t)))
-        else:
-            best, why = t, ("the two quads aren't nested; tone's band assumption "
-                            "holding is itself evidence that it found a screen")
-    elif t is None and sat is not None:
-        # tone was dropped for having sharp corners, or never fired. The
-        # surviving region detector measured a real corner radius; edge usually
-        # cannot -- but when it did, and more confidently, it wins the same way.
-        if e is not None and shape_tier(e) > shape_tier(sat):
-            best, why = e, ("the edge quad's corners agree on a radius (tier %d) "
-                            "where saturation's do not (tier %d)"
-                            % (shape_tier(e), shape_tier(sat)))
+                            "corners agree on a radius (tier %d) where %s's do "
+                            "not (tier %d)" % (te, rname, tr))
         else:
-            best, why = sat, ("the saturation detector found a region with rounded "
-                              "corners where tone did not")
+            best, why = region, ("the two quads aren't nested; %s's assumption "
+                                 "holding is itself evidence that it found a screen"
+                                 % rname)
     else:
         best = t or e or sat
     if best is None:

package/skills/inject-screenshot/SKILL.md

@@ -5,7 +5,7 @@ description: Injects a UI screenshot OR a screen recording onto a photographed d
 
 # Inject a screenshot onto a photographed device
 
-**What ships (v0.38):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screen source, which may be an image **or a video** (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position — and when detection cannot tell which region is a screen, **Point at screen**: one click inside it and the detector uses that point — or, if this photograph has been fitted before, **the fit it was saved with comes back** as the starting position instead of a detection, recognised by the photo's own pixels so a rename or a drag-drop still match — and every save also writes a **portable `.fit.json` beside the mockup** that can be dropped back onto the page later, which is how a fit survives a re-export, another machine, or someone else's hands — then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas navigation that follows the usual conventions — **hold ⌘ and scroll to zoom to the pointer, hold space and drag to pan** — and a rectified strip loupe. The fit and the composite sit **side by side and always have** — the result pane re-renders as you drag, which is how a corner gets judged, so it is the layout rather than a mode you can switch off. Then an on-by-default realism pass that colour-matches the source to the photo's light, **Save** (or **Render**, for a video) into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
+**What ships (v0.40):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screen source, which may be an image **or a video** (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position — and when detection cannot tell which region is a screen, **Point at screen**: one click inside it and the detector uses that point — or, if this photograph has been fitted before, **the fit it was saved with comes back** as the starting position instead of a detection, recognised by the photo's own pixels so a rename or a drag-drop still match — and every save also writes a **portable `.fit.json` beside the mockup** that can be dropped back onto the page later, which is how a fit survives a re-export, another machine, or someone else's hands — then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas navigation that follows the usual conventions — **hold ⌘ and scroll to zoom to the pointer, hold space and drag to pan** — and a rectified strip loupe. The fit and the composite sit **side by side and always have** — the result pane re-renders as you drag, which is how a corner gets judged, so it is the layout rather than a mode you can switch off. Then an on-by-default realism pass that colour-matches the source to the photo's light, **Save** (or **Render**, for a video) into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
 
 The geometry is exact (`warp.py`); the detection is advisory (`detect.py`) and the human corrects it. **When a detection is wrong and you want to know why**, ask for the candidate list: `POST /api/detect {"trace": true}` writes `<session>/candidates.json`, or run `python3 scripts/detect.py --photo P --out-corners /tmp/c.json --trace /tmp/t.json` (add `--click X,Y`). Every candidate quad is in there with its score and whether it was accepted, rejected, never reached, or filtered out by the click — which is what separates "the screen was never proposed" from "it was proposed and something else won".