screengraft 0.37.0 → 0.39.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screengraft",
-  "version": "0.37.0",
+  "version": "0.39.0",
   "description": "Put a UI screenshot or screen recording onto a photographed device screen with the perspective exactly right \u2014 a homography you confirm by hand, not a generative guess.",
   "keywords": [
     "mockup",

package/scripts/detect.py

@@ -875,6 +875,26 @@ def detect_saturation(bgr: np.ndarray, click=None, trace=None):
     return res
 
 
+def shape_tier(result) -> int:
+    """How strongly a result's own SHAPE says "this is a screen": 0, 1 or 2.
+
+        2  confident radius -- four per-corner estimates agree within 50%
+        1  rounded          -- they agree within MAX_RADIUS_SPREAD (2x)
+        0  nothing          -- sharp, unmeasurable, or wildly inconsistent
+
+    Both thresholds already existed (measure_corner_radius's `confident`, and
+    has_rounded_corners); this only ranks them. Measured 11 Sep 2026 on ten
+    hand-labelled photographs, every channel result: **every tier-2 quad was on
+    the screen** (worst spread 0.24) and **every wrong quad was >= 1.43 or
+    unmeasurable** -- a 6x margin. Tier 1 alone does not separate them (a
+    correct tone quad at 1.42 against a wrong one at 1.43), which is exactly why
+    a tier-1 result must not outrank or veto a tier-2 one. It was doing both.
+    """
+    if (result.get("corner_radius") or {}).get("confident"):
+        return 2
+    return 1 if has_rounded_corners(result) else 0
+
+
 def has_rounded_corners(result) -> bool:
     """Did this quad's own outline actually curve at the corners?
 
@@ -939,7 +959,16 @@ def detect(gray: np.ndarray, tone=None, method="auto", color=None, click=None,
 
     if not results:
         return None
+    return arbitrate(results, gray.shape[:2], click)
 
+
+def arbitrate(results, shape, click=None):
+    """Choose among the channels' accepted results and decide whether to abstain.
+
+    Split out of detect() on 11 Sep 2026 so the rules here can be tested on
+    hand-built results, the way _finalize's nested pick already is. `shape` is
+    the photograph's (h, w); the only thing this needs it for is the diagonal.
+    """
     # tone and saturation are the same algorithm on different channels, so they
     # are compared to each other before anything else, on whether the region
     # each found actually has rounded corners. A patch of table cut out of a
@@ -967,27 +996,76 @@ def detect(gray: np.ndarray, tone=None, method="auto", color=None, click=None,
     sat = next((r for r in results if r["method"] == "saturation"), None)
     why = "it was the only detector left after the rounded-corner filter" \
         if len(results) == 1 else "its tone band assumption held, which is itself evidence"
-    if t and e:
-        tq, eq = t["_corners_np"], e["_corners_np"]
-        ta = float(cv2.contourArea(tq.astype(np.float32)))
+    # ONE region-vs-edge arbitration, for whichever region channel survived the
+    # rounded-corner filter above. Until 11 Sep 2026 only tone got the nesting
+    # rules; saturation-vs-edge fell through to "saturation wins". On iPhone-4
+    # that returned a saturation quad 18% off, nested around an edge quad 3%
+    # off at 94% of its area -- exactly the "screen inside a body" shape the
+    # tone branch already knew how to read.
+    region = t if t is not None else sat
+    rname = "tone" if t is not None else "saturation"
+    if region is not None and e is not None:
+        rq, eq = region["_corners_np"], e["_corners_np"]
+        ra = float(cv2.contourArea(rq.astype(np.float32)))
         ea = float(cv2.contourArea(eq.astype(np.float32)))
-        nested = overlap_frac(tq, eq) >= 0.90 and ta < ea
-        ratio = ta / ea if ea > 0 else 0.0
-        if nested and ratio < NEST_FLOOR:
-            best, why = e, ("the tone quad is only %.0f%% of the edge quad it sits "
-                            "inside — that's content drawn on the screen, not the "
-                            "screen" % (ratio * 100))
+        # Nesting is read on whichever quad is inside the other. It used to be
+        # read only with the region quad inside the edge quad, so an edge quad
+        # sitting inside a region quad (iPhone-4: edge 3% off inside a
+        # saturation quad 18% off at 94%) was "not nested" and fell through to
+        # the region winning by default.
+        inner, outer = (region, e) if ra < ea else (e, region)
+        iname, oname = (rname, "edge") if ra < ea else ("edge", rname)
+        nested = overlap_frac(inner["_corners_np"], outer["_corners_np"]) >= 0.90
+        ratio = min(ra, ea) / max(ra, ea) if max(ra, ea) > 0 else 0.0
+        ti, to = shape_tier(inner), shape_tier(outer)
+        tr, te = shape_tier(region), shape_tier(e)
+        # Nested: shape evidence first, the area ratio only to break a tie.
+        #
+        # The ratio alone reads "small inside big" as content-inside-screen and
+        # "large inside big" as screen-inside-body, and it is right on the
+        # gradient-screen fixture (41%: a slab on the screen) and on most
+        # photographs. It is wrong exactly when the quads' own corners say the
+        # opposite: a UI content region at 96% of the screen with loose corners
+        # (two of ten labelled photographs, 15% off) is not a screen inside a
+        # body, and a confident phone screen at 34% of a loosely-rounded table
+        # (the hand-built case in test_detect) is not content on it. Tiers
+        # decide those; at equal tiers the ratio still does, unchanged.
+        if nested and to > ti:
+            best, why = outer, ("the %s quad sits inside the %s quad at %.0f%% of its "
+                                "area, but the outer quad's corners agree on a radius "
+                                "(tier %d) and the inner's do not (tier %d) — content "
+                                "inside a screen, not a screen inside a body"
+                                % (iname, oname, ratio * 100, to, ti))
+        elif nested and ti > to:
+            best, why = inner, ("the %s quad sits inside the %s quad at %.0f%% of its "
+                                "area, and the inner quad's corners agree on a radius "
+                                "(tier %d) where the outer's do not (tier %d) — a "
+                                "screen on something larger"
+                                % (iname, oname, ratio * 100, ti, to))
+        elif nested and ratio < NEST_FLOOR:
+            best, why = outer, ("the %s quad is only %.0f%% of the %s quad it sits "
+                                "inside — that's content drawn on the screen, not the "
+                                "screen" % (iname, ratio * 100, oname))
         elif nested:
-            best, why = t, ("the tone quad sits inside the edge quad at %.0f%% of "
-                            "its area — a screen inside a device body" % (ratio * 100))
+            best, why = inner, ("the %s quad sits inside the %s quad at %.0f%% of "
+                                "its area — a screen inside a device body"
+                                % (iname, oname, ratio * 100))
+        elif te > tr:
+            # Not nested, and the edge quad's own shape says "screen" more
+            # strongly than the region's does. Before 11 Sep 2026 tone won here
+            # unconditionally, on the argument that its band assumption holding
+            # was itself evidence -- and on two of ten labelled photographs that
+            # handed the answer to a tone quad 159% and 249% off, with corner
+            # spreads of 3.4 and 19.9, over an edge quad at 0% with a confident
+            # radius. A band assumption is weaker evidence than four agreeing
+            # corners; the tiers say so and this reads them.
+            best, why = e, ("the two quads aren't nested and the edge quad's "
+                            "corners agree on a radius (tier %d) where %s's do "
+                            "not (tier %d)" % (te, rname, tr))
         else:
-            best, why = t, ("the two quads aren't nested; tone's band assumption "
-                            "holding is itself evidence that it found a screen")
-    elif t is None and sat is not None:
-        # tone was dropped for having sharp corners, or never fired. The
-        # surviving region detector measured a real corner radius; edge cannot.
-        best, why = sat, ("the saturation detector found a region with rounded "
-                          "corners where tone did not")
+            best, why = region, ("the two quads aren't nested; %s's assumption "
+                                 "holding is itself evidence that it found a screen"
+                                 % rname)
     else:
         best = t or e or sat
     if best is None:
@@ -999,7 +1077,7 @@ def detect(gray: np.ndarray, tone=None, method="auto", color=None, click=None,
         # somewhere else entirely should not erase the fact that two of them
         # landed together. Corroboration by any one independent method is the
         # evidence worth reporting.
-        diag = float(np.hypot(*gray.shape[:2]))
+        diag = float(np.hypot(*shape))
         others = [r for r in results if r is not best]
         gaps = [(float(np.max(np.linalg.norm(best["_corners_np"]
                                              - r["_corners_np"], axis=1))) / diag, r)
@@ -1047,9 +1125,15 @@ def detect(gray: np.ndarray, tone=None, method="auto", color=None, click=None,
     # not get a vote on whether the rounded thing is a screen. So the gap is
     # re-measured against peers that also found something screen-shaped; when
     # there are none, being alone is not evidence of being wrong.
-    peers = [r for r in results if r is not best and has_rounded_corners(r)]
+    # ... and a peer may only veto a result whose shape evidence it at least
+    # matches. On iPhone-2 (11 Sep 2026) a tone quad 143% off, "rounded" at a
+    # spread of 1.43, vetoed a confident edge quad 0.3% off -- the bench's one
+    # "good quad refused". A veto from weaker evidence is not a disagreement
+    # between peers; it is noise outvoting a measurement.
+    peers = [r for r in results if r is not best and has_rounded_corners(r)
+             and shape_tier(r) >= shape_tier(best)]
     if peers:
-        diag = float(np.hypot(*gray.shape[:2]))
+        diag = float(np.hypot(*shape))
         peer_gap = min(float(np.max(np.linalg.norm(best["_corners_np"]
                                                    - r["_corners_np"], axis=1))) / diag
                        for r in peers)

package/skills/inject-screenshot/SKILL.md

@@ -5,7 +5,7 @@ description: Injects a UI screenshot OR a screen recording onto a photographed d
 
 # Inject a screenshot onto a photographed device
 
-**What ships (v0.37):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screen source, which may be an image **or a video** (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position — and when detection cannot tell which region is a screen, **Point at screen**: one click inside it and the detector uses that point — or, if this photograph has been fitted before, **the fit it was saved with comes back** as the starting position instead of a detection, recognised by the photo's own pixels so a rename or a drag-drop still match — and every save also writes a **portable `.fit.json` beside the mockup** that can be dropped back onto the page later, which is how a fit survives a re-export, another machine, or someone else's hands — then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas navigation that follows the usual conventions — **hold ⌘ and scroll to zoom to the pointer, hold space and drag to pan** — and a rectified strip loupe. The fit and the composite sit **side by side and always have** — the result pane re-renders as you drag, which is how a corner gets judged, so it is the layout rather than a mode you can switch off. Then an on-by-default realism pass that colour-matches the source to the photo's light, **Save** (or **Render**, for a video) into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
+**What ships (v0.39):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screen source, which may be an image **or a video** (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position — and when detection cannot tell which region is a screen, **Point at screen**: one click inside it and the detector uses that point — or, if this photograph has been fitted before, **the fit it was saved with comes back** as the starting position instead of a detection, recognised by the photo's own pixels so a rename or a drag-drop still match — and every save also writes a **portable `.fit.json` beside the mockup** that can be dropped back onto the page later, which is how a fit survives a re-export, another machine, or someone else's hands — then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas navigation that follows the usual conventions — **hold ⌘ and scroll to zoom to the pointer, hold space and drag to pan** — and a rectified strip loupe. The fit and the composite sit **side by side and always have** — the result pane re-renders as you drag, which is how a corner gets judged, so it is the layout rather than a mode you can switch off. Then an on-by-default realism pass that colour-matches the source to the photo's light, **Save** (or **Render**, for a video) into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
 
 The geometry is exact (`warp.py`); the detection is advisory (`detect.py`) and the human corrects it. **When a detection is wrong and you want to know why**, ask for the candidate list: `POST /api/detect {"trace": true}` writes `<session>/candidates.json`, or run `python3 scripts/detect.py --photo P --out-corners /tmp/c.json --trace /tmp/t.json` (add `--click X,Y`). Every candidate quad is in there with its score and whether it was accepted, rejected, never reached, or filtered out by the click — which is what separates "the screen was never proposed" from "it was proposed and something else won".