screengraft 0.43.0 → 0.45.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screengraft",
-  "version": "0.43.0",
+  "version": "0.45.0",
   "description": "Put a UI screenshot or screen recording onto a photographed device screen with the perspective exactly right \u2014 a homography you confirm by hand, not a generative guess.",
   "keywords": [
     "mockup",

package/scripts/detect.py

@@ -130,6 +130,12 @@ ABSTAIN_GAP = 0.15
 # small part of it. That one ratio separates "step inward to the screen" from
 # "don't step into a panel", and it arbitrates between the two detectors too.
 NEST_FLOOR = 0.55
+# "Nested" means this much of the inner quad's area falls inside the outer.
+# One number for every place that asks (arbitrate, the walk order, the
+# abstention veto) so they cannot drift apart. 12 Sep 2026: on 18 labelled
+# photographs the quads that are nested overlap at >= 0.95 and the ones that
+# are not at 0.00 -- there is nothing in between to be careful about.
+NESTED_OVERLAP = 0.90
 
 
 def _odd(n: int) -> int:
@@ -421,7 +427,7 @@ def overlap_frac(inner: np.ndarray, outer: np.ndarray) -> float:
     return float(cv2.contourArea(region.astype(np.float32))) / a
 
 
-def pick_innermost(candidates, best):
+def pick_innermost(candidates, best, quad_of=None):
     """
     A device photo offers more than one screen-shaped region: the glass screen,
     and the bezel or body it sits in. They nest, and the outer one wins on
@@ -436,20 +442,33 @@ def pick_innermost(candidates, best):
     old 0.35 floor a wave-shaped gradient at 49% of its screen was eligible;
     at 0.55 it isn't (an earlier finding, 3 Sep 2026).
     """
+    # `quad_of` maps a candidate to the quad its nesting is judged on. The
+    # walk passes the REFINED corners; the default is the raw polygon
+    # approximation. Raw vertices sit on the corner arcs, not at the corners,
+    # and on a phone photographed at ~45 degrees the glass's raw vertex landed
+    # ON the body's raw edge line -- `quad_contains` was false, and the body
+    # (tier 2, 13% off) shipped with the glass (tier 2, 1%, 87% of its area)
+    # one step behind it (12 Sep 2026, a two-phone mockup). Judged on the
+    # refined corners the glass is 10px inside the body all round. Replacing
+    # the containment test with area overlap was tried first and stepped into
+    # wrong inner quads on five photographs -- the strict test is doing work.
+    if quad_of is None:
+        quad_of = lambda c: c[1]  # noqa: E731
     current = best
     for _ in range(4):  # screen inside bezel inside body: a few steps is plenty
-        c_area = cv2.contourArea(current[1].astype(np.float32))
+        cq = quad_of(current)
+        c_area = cv2.contourArea(cq.astype(np.float32))
         inner = [
             c for c in candidates
             if c is not current
             and c[0] >= 0.25 * current[0]
-            and quad_contains(current[1], c[1])
-            and NEST_FLOOR * c_area <= cv2.contourArea(c[1].astype(np.float32)) < c_area
+            and quad_contains(cq, quad_of(c))
+            and NEST_FLOOR * c_area <= cv2.contourArea(quad_of(c).astype(np.float32)) < c_area
         ]
         if not inner:
             return current
         # Largest of the nested ones: the screen, not a panel drawn on it.
-        current = max(inner, key=lambda c: cv2.contourArea(c[1].astype(np.float32)))
+        current = max(inner, key=lambda c: cv2.contourArea(quad_of(c).astype(np.float32)))
     return current
 
 
@@ -635,34 +654,67 @@ def _finalize(candidates, img_area: float, refine: bool = True, img_shape=None,
     # project has never had the number. Off unless asked for, and it must not
     # change the answer: it observes the same lists the walk below uses.
     walked = {}
-    # Walk order: by score -- EXCEPT when the best-scoring candidate has no
-    # shape evidence at all (tier 0: sharp, unmeasurable, or inconsistent
-    # corners) while a tier-2 candidate exists in the same list. A tier-0
-    # winner cannot be corroborated and ends in an abstention or a refusal, so
-    # nothing is lost by looking past it; a tier-2 candidate has four corners
-    # agreeing on a radius, which on the labelled corpus has meant "on the
-    # screen" for every channel-accepted result. Deliberately NOT tier-first
-    # ordering of the whole list: that was tried on 11 Sep 2026 and broke three
-    # photographs, because a tier-2 WRONG candidate sits deeper in the list on
-    # each. This only moves when score alone would have handed the answer to a
-    # quad with nothing to say for itself.
-    def _tier_of(cand):
-        _score, quad, contour, _tag = cand
-        q = order_quad(refine_corners(contour, quad, rail_band)[0]) if refine else quad
-        return shape_tier({"corner_radius": measure_corner_radius(contour, q)})
+    # Walk order: by score -- EXCEPT when the best-scoring candidate is not
+    # itself confident (tier < 2) while a tier-2 candidate exists in the same
+    # list. Two cases, and they are different:
+    #
+    # * Top is tier 0 (sharp, unmeasurable, or inconsistent corners): it cannot
+    #   be corroborated and ends in an abstention or a refusal, so nothing is
+    #   lost by looking past it to ANY tier-2 candidate.
+    # * Top is tier 1 (rounded, but the four radii disagree): it is a real
+    #   finding, and only a tier-2 candidate NESTED INSIDE it may move ahead --
+    #   the same finding narrowed to the part whose four corners agree, which
+    #   is how arbitrate() already reads a confident quad inside a loosely
+    #   rounded one (tiers before area ratio). A tier-2 candidate somewhere
+    #   ELSE is a different finding and does not displace a rounded top on
+    #   tier alone. Measured 12 Sep 2026 on 18 labelled photographs: every
+    #   tier-2 that should jump a tier-1 top lies inside it at >= 0.95
+    #   overlap; every one that must not (tone's patches on two photographs,
+    #   an edge quad on a third) overlaps it at 0.00. Without the nesting
+    #   condition the tone channel promoted a disjoint tier-2 patch 202% off
+    #   on two photographs and the answer was lost to abstention. The case
+    #   that needed this is a front-and-back mockup: the edge channel's top
+    #   candidate is the bounding box of BOTH phones (tier 1, 105% off), and
+    #   the screen sits inside it at 48% of its area -- under NEST_FLOOR, so
+    #   pick_innermost cannot step to it on the ratio.
+    #
+    # Deliberately NOT tier-first ordering of the whole list: that was tried
+    # on 11 Sep 2026 and broke three photographs, because a tier-2 WRONG
+    # candidate sits deeper in the list on each.
+    # Refined corners, once per candidate: the walk order reads tiers off them
+    # and pick_innermost judges nesting on them (raw polygon vertices sit on
+    # the arcs and can land on a neighbouring quad's edge line -- see there).
+    shapes = {}
+
+    def _shape_of(cand):
+        if id(cand) not in shapes:
+            _score, quad, contour, _tag = cand
+            q = order_quad(refine_corners(contour, quad, rail_band)[0]) if refine else quad
+            shapes[id(cand)] = (q, shape_tier({"corner_radius": measure_corner_radius(contour, q)}))
+        return shapes[id(cand)]
+
+    def _refined(cand):
+        return _shape_of(cand)[0]
 
     order = sorted(candidates, key=lambda c: c[0], reverse=True)
-    if order and _tier_of(order[0]) == 0:
-        # Identity, not equality: candidates hold numpy arrays.
-        strong = [c for c in order if _tier_of(c) == 2]
-        if strong:
-            strong_ids = {id(c) for c in strong}
-            order = strong + [c for c in order if id(c) not in strong_ids]
+    if order:
+        top_q, top_tier = _shape_of(order[0])
+        if top_tier < 2:
+            strong = []
+            for c in order[1:]:
+                q, tier = _shape_of(c)
+                if tier == 2 and (top_tier == 0
+                                  or overlap_frac(q, top_q) >= NESTED_OVERLAP):
+                    strong.append(c)
+            if strong:
+                # Identity, not equality: candidates hold numpy arrays.
+                strong_ids = {id(c) for c in strong}
+                order = strong + [c for c in order if id(c) not in strong_ids]
     for cand in order:
         # NOT necessarily `cand`: pick_innermost steps inward from it while a
         # comparably screen-like quad nests inside, so the quad that gets
         # validated -- and returned -- can belong to a different candidate.
-        picked = pick_innermost(candidates, cand)
+        picked = pick_innermost(candidates, cand, quad_of=_refined)
         score, quad, contour, tag = picked
         if refine:
             refined, did_refine = refine_corners(contour, quad, rail_band)
@@ -753,7 +805,13 @@ def _walk_rows(generated, survived, walked, method, click, refine=True,
         else:
             verdict, why = walked.get(id(cand), ("unreached",
                                                  "a higher-scoring candidate was accepted first"))
-        rows.append(_trace_row(method, score, quad, tag, verdict, why))
+        row = _trace_row(method, score, quad, tag, verdict, why)
+        # The shape tier of every proposal, not only the winner's: the walk
+        # order rule in _finalize reads tiers, so a bench reading this file
+        # has to be able to see what the walk saw (added 12 Sep 2026 while
+        # tracing a 105% edge win with a 5% tier-2 candidate right behind it).
+        row["tier"] = shape_tier({"corner_radius": measure_corner_radius(contour, quad)})
+        rows.append(row)
     if click is not None:
         for r in rows:
             r["contains_click"] = r["verdict"] != "filtered_by_click"
@@ -1069,7 +1127,7 @@ def arbitrate(results, shape, click=None):
         # the region winning by default.
         inner, outer = (region, e) if ra < ea else (e, region)
         iname, oname = (rname, "edge") if ra < ea else ("edge", rname)
-        nested = overlap_frac(inner["_corners_np"], outer["_corners_np"]) >= 0.90
+        nested = overlap_frac(inner["_corners_np"], outer["_corners_np"]) >= NESTED_OVERLAP
         ratio = min(ra, ea) / max(ra, ea) if max(ra, ea) > 0 else 0.0
         ti, to = shape_tier(inner), shape_tier(outer)
         tr, te = shape_tier(region), shape_tier(e)
@@ -1184,8 +1242,17 @@ def arbitrate(results, shape, click=None):
     # spread of 1.43, vetoed a confident edge quad 0.3% off -- the bench's one
     # "good quad refused". A veto from weaker evidence is not a disagreement
     # between peers; it is noise outvoting a measurement.
+    # ... and a peer sitting INSIDE the winner is not disagreeing about where
+    # the screen is. On iPhone-15 (12 Sep 2026) arbitration had just ruled a
+    # tone quad to be content drawn on the edge quad's screen -- nested at 9%
+    # of its area, a rounded card -- and this gate then read that same card as
+    # a credible second opinion 32% of the diagonal away and abstained, holding
+    # a quad 0% from the label. A veto is for two screen-shaped findings in
+    # two PLACES; a card on the screen is one place. The nested case has
+    # already been decided above, on tiers and ratio, by the time this runs.
     peers = [r for r in results if r is not best and has_rounded_corners(r)
-             and shape_tier(r) >= shape_tier(best)]
+             and shape_tier(r) >= shape_tier(best)
+             and overlap_frac(r["_corners_np"], best["_corners_np"]) < NESTED_OVERLAP]
     if peers:
         diag = float(np.hypot(*shape))
         peer_gap = min(float(np.max(np.linalg.norm(best["_corners_np"]

package/skills/inject-screenshot/SKILL.md

@@ -5,7 +5,7 @@ description: Injects a UI screenshot OR a screen recording onto a photographed d
 
 # Inject a screenshot onto a photographed device
 
-**What ships (v0.43):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screen source, which may be an image **or a video** (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position — and when detection cannot tell which region is a screen, **Point at screen**: one click inside it and the detector uses that point — or, if this photograph has been fitted before, **the fit it was saved with comes back** as the starting position instead of a detection, recognised by the photo's own pixels so a rename or a drag-drop still match — and every save also writes a **portable `.fit.json` beside the mockup** that can be dropped back onto the page later, which is how a fit survives a re-export, another machine, or someone else's hands — then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas navigation that follows the usual conventions — **hold ⌘ and scroll to zoom to the pointer, hold space and drag to pan** — and a rectified strip loupe. The fit and the composite sit **side by side and always have** — the result pane re-renders as you drag, which is how a corner gets judged, so it is the layout rather than a mode you can switch off. Then an on-by-default realism pass that colour-matches the source to the photo's light, **Save** (or **Render**, for a video) into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
+**What ships (v0.45):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screen source, which may be an image **or a video** (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position — and when detection cannot tell which region is a screen, **Point at screen**: one click inside it and the detector uses that point — or, if this photograph has been fitted before, **the fit it was saved with comes back** as the starting position instead of a detection, recognised by the photo's own pixels so a rename or a drag-drop still match — and every save also writes a **portable `.fit.json` beside the mockup** that can be dropped back onto the page later, which is how a fit survives a re-export, another machine, or someone else's hands — then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas navigation that follows the usual conventions — **hold ⌘ and scroll to zoom to the pointer, hold space and drag to pan** — and a rectified strip loupe. The fit and the composite sit **side by side and always have** — the result pane re-renders as you drag, which is how a corner gets judged, so it is the layout rather than a mode you can switch off. Then an on-by-default realism pass that colour-matches the source to the photo's light, **Save** (or **Render**, for a video) into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
 
 The geometry is exact (`warp.py`); the detection is advisory (`detect.py`) and the human corrects it. **When a detection is wrong and you want to know why**, ask for the candidate list: `POST /api/detect {"trace": true}` writes `<session>/candidates.json`, or run `python3 scripts/detect.py --photo P --out-corners /tmp/c.json --trace /tmp/t.json` (add `--click X,Y`). Every candidate quad is in there with its score and whether it was accepted, rejected, never reached, or filtered out by the click — which is what separates "the screen was never proposed" from "it was proposed and something else won".