screengraft 0.44.0 → 0.46.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screengraft",
-  "version": "0.44.0",
+  "version": "0.46.0",
   "description": "Put a UI screenshot or screen recording onto a photographed device screen with the perspective exactly right \u2014 a homography you confirm by hand, not a generative guess.",
   "keywords": [
     "mockup",

package/scripts/detect.py

@@ -260,10 +260,22 @@ def approx_quad(contour):
 
 
 def order_quad(pts: np.ndarray) -> np.ndarray:
-    """Order 4 points TL, TR, BR, BL as they appear in the image."""
+    """Order 4 points TL, TR, BR, BL as they appear in the IMAGE.
+
+    Corner 0 is where the screenshot's top-left lands, so this decides the
+    screenshot's orientation -- and this function cannot decide it well: on a
+    phone photographed at ~45 degrees the corner nearest the image's top-left
+    is the screen's physical bottom-left, so a quad right to 1% put the
+    screenshot a quarter turn out (12 Sep 2026). A rule "the top edge of
+    a portrait quad is its higher short edge" was tried here and is wrong for
+    every landscape screen, because which edge is the top depends on the
+    SCREENSHOT's aspect, which only the workbench knows. So this stays
+    geometric and the workbench turns the order to match the screenshot
+    (`orientQuad` in ui/index.html, plus a Rotate 90 degrees control).
+    """
     c = pts.mean(axis=0)
     ang = np.arctan2(pts[:, 1] - c[1], pts[:, 0] - c[0])
-    pts = pts[np.argsort(ang)]           # counter-clockwise in image coords
+    pts = pts[np.argsort(ang)]           # clockwise as seen on screen (y down)
     start = int(np.argmin(pts.sum(axis=1)))  # closest to the image's top-left
     return np.roll(pts, -start, axis=0)
 
@@ -427,7 +439,7 @@ def overlap_frac(inner: np.ndarray, outer: np.ndarray) -> float:
     return float(cv2.contourArea(region.astype(np.float32))) / a
 
 
-def pick_innermost(candidates, best):
+def pick_innermost(candidates, best, quad_of=None):
     """
     A device photo offers more than one screen-shaped region: the glass screen,
     and the bezel or body it sits in. They nest, and the outer one wins on
@@ -442,20 +454,33 @@ def pick_innermost(candidates, best):
     old 0.35 floor a wave-shaped gradient at 49% of its screen was eligible;
     at 0.55 it isn't (an earlier finding, 3 Sep 2026).
     """
+    # `quad_of` maps a candidate to the quad its nesting is judged on. The
+    # walk passes the REFINED corners; the default is the raw polygon
+    # approximation. Raw vertices sit on the corner arcs, not at the corners,
+    # and on a phone photographed at ~45 degrees the glass's raw vertex landed
+    # ON the body's raw edge line -- `quad_contains` was false, and the body
+    # (tier 2, 13% off) shipped with the glass (tier 2, 1%, 87% of its area)
+    # one step behind it (12 Sep 2026, a two-phone mockup). Judged on the
+    # refined corners the glass is 10px inside the body all round. Replacing
+    # the containment test with area overlap was tried first and stepped into
+    # wrong inner quads on five photographs -- the strict test is doing work.
+    if quad_of is None:
+        quad_of = lambda c: c[1]  # noqa: E731
     current = best
     for _ in range(4):  # screen inside bezel inside body: a few steps is plenty
-        c_area = cv2.contourArea(current[1].astype(np.float32))
+        cq = quad_of(current)
+        c_area = cv2.contourArea(cq.astype(np.float32))
         inner = [
             c for c in candidates
             if c is not current
             and c[0] >= 0.25 * current[0]
-            and quad_contains(current[1], c[1])
-            and NEST_FLOOR * c_area <= cv2.contourArea(c[1].astype(np.float32)) < c_area
+            and quad_contains(cq, quad_of(c))
+            and NEST_FLOOR * c_area <= cv2.contourArea(quad_of(c).astype(np.float32)) < c_area
         ]
         if not inner:
             return current
         # Largest of the nested ones: the screen, not a panel drawn on it.
-        current = max(inner, key=lambda c: cv2.contourArea(c[1].astype(np.float32)))
+        current = max(inner, key=lambda c: cv2.contourArea(quad_of(c).astype(np.float32)))
     return current
 
 
@@ -668,10 +693,20 @@ def _finalize(candidates, img_area: float, refine: bool = True, img_shape=None,
     # Deliberately NOT tier-first ordering of the whole list: that was tried
     # on 11 Sep 2026 and broke three photographs, because a tier-2 WRONG
     # candidate sits deeper in the list on each.
+    # Refined corners, once per candidate: the walk order reads tiers off them
+    # and pick_innermost judges nesting on them (raw polygon vertices sit on
+    # the arcs and can land on a neighbouring quad's edge line -- see there).
+    shapes = {}
+
     def _shape_of(cand):
-        _score, quad, contour, _tag = cand
-        q = order_quad(refine_corners(contour, quad, rail_band)[0]) if refine else quad
-        return q, shape_tier({"corner_radius": measure_corner_radius(contour, q)})
+        if id(cand) not in shapes:
+            _score, quad, contour, _tag = cand
+            q = order_quad(refine_corners(contour, quad, rail_band)[0]) if refine else quad
+            shapes[id(cand)] = (q, shape_tier({"corner_radius": measure_corner_radius(contour, q)}))
+        return shapes[id(cand)]
+
+    def _refined(cand):
+        return _shape_of(cand)[0]
 
     order = sorted(candidates, key=lambda c: c[0], reverse=True)
     if order:
@@ -691,7 +726,7 @@ def _finalize(candidates, img_area: float, refine: bool = True, img_shape=None,
         # NOT necessarily `cand`: pick_innermost steps inward from it while a
         # comparably screen-like quad nests inside, so the quad that gets
         # validated -- and returned -- can belong to a different candidate.
-        picked = pick_innermost(candidates, cand)
+        picked = pick_innermost(candidates, cand, quad_of=_refined)
         score, quad, contour, tag = picked
         if refine:
             refined, did_refine = refine_corners(contour, quad, rail_band)

package/skills/inject-screenshot/SKILL.md

@@ -5,7 +5,7 @@ description: Injects a UI screenshot OR a screen recording onto a photographed d
 
 # Inject a screenshot onto a photographed device
 
-**What ships (v0.44):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screen source, which may be an image **or a video** (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position — and when detection cannot tell which region is a screen, **Point at screen**: one click inside it and the detector uses that point — or, if this photograph has been fitted before, **the fit it was saved with comes back** as the starting position instead of a detection, recognised by the photo's own pixels so a rename or a drag-drop still match — and every save also writes a **portable `.fit.json` beside the mockup** that can be dropped back onto the page later, which is how a fit survives a re-export, another machine, or someone else's hands — then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas navigation that follows the usual conventions — **hold ⌘ and scroll to zoom to the pointer, hold space and drag to pan** — and a rectified strip loupe. The fit and the composite sit **side by side and always have** — the result pane re-renders as you drag, which is how a corner gets judged, so it is the layout rather than a mode you can switch off. Then an on-by-default realism pass that colour-matches the source to the photo's light, **Save** (or **Render**, for a video) into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
+**What ships (v0.46):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screen source, which may be an image **or a video** (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position — and when detection cannot tell which region is a screen, **Point at screen**: one click inside it and the detector uses that point — or, if this photograph has been fitted before, **the fit it was saved with comes back** as the starting position instead of a detection, recognised by the photo's own pixels so a rename or a drag-drop still match — and every save also writes a **portable `.fit.json` beside the mockup** that can be dropped back onto the page later, which is how a fit survives a re-export, another machine, or someone else's hands — then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas navigation that follows the usual conventions — **hold ⌘ and scroll to zoom to the pointer, hold space and drag to pan** — and a rectified strip loupe. The fit and the composite sit **side by side and always have** — the result pane re-renders as you drag, which is how a corner gets judged, so it is the layout rather than a mode you can switch off. Then an on-by-default realism pass that colour-matches the source to the photo's light, **Save** (or **Render**, for a video) into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
 
 The geometry is exact (`warp.py`); the detection is advisory (`detect.py`) and the human corrects it. **When a detection is wrong and you want to know why**, ask for the candidate list: `POST /api/detect {"trace": true}` writes `<session>/candidates.json`, or run `python3 scripts/detect.py --photo P --out-corners /tmp/c.json --trace /tmp/t.json` (add `--click X,Y`). Every candidate quad is in there with its score and whether it was accepted, rejected, never reached, or filtered out by the click — which is what separates "the screen was never proposed" from "it was proposed and something else won".
 

package/ui/index.html

@@ -984,6 +984,7 @@
           <span class="stepper"><button class="sm" data-jump="0">TL</button><button class="sm" data-jump="1">TR</button><button class="sm" data-jump="2">BR</button><button class="sm" data-jump="3">BL</button></span>
           <button class="sm" id="redetect">Re-detect</button>
           <button class="sm" id="pointat" title="Click once inside the screen and the detector will use that point. The detectors usually do find the screen — they just cannot tell which region IS one, and that is the part you can answer instantly.">Point at screen</button>
+          <button class="sm" id="rotatequad" title="Turn the screenshot a quarter turn inside the same four edges. The edges stay where they are; only which one is the top changes. Use this when the phone lies on its side and the screenshot lands sideways.">Rotate 90°</button>
           <button class="sm" id="resetquad" title="Put the four edges back to a rectangle in the middle of the photo. Use this if a corner has ended up off the picture where you cannot grab it.">Reset</button>
         </span>
       </div>
@@ -1568,7 +1569,10 @@ function setChosen(role, path, size, el, meta){
   // screenshot to put inside it, which is what maybeStart() waits for.
   if (role==='photo'){ st.photo = path; st.corners = null;
     st.remembered = (meta && meta.remembered) || null; }
-  else { st.shot = path; st.video = meta && meta.video ? meta : null; syncVideoUI(); }
+  else { st.shot = path; st.shotSize = size; st.video = meta && meta.video ? meta : null; syncVideoUI();
+    // A detected quad is re-oriented for the new screenshot; a remembered or
+    // hand-placed one is somebody's decision and is left alone.
+    if (st.corners && st.quadFrom === 'detected' && orientQuad()){ draw(); drawStrip(); autoPreview(); } }
   closePop();
   maybeStart();
 }
@@ -1931,6 +1935,7 @@ async function pointAt(x, y){
     const r = await api('/api/detect', {click: [x, y]});
     if (r.found){
       st.corners = r.corners; st.measured = r.corner_radius; st.quadFrom = 'detected';
+      orientQuad();
       s.className = 'status ok';
       s.textContent = `Screen found from your click (${r.method})`;
       if (!st.type) setType(r.type_guess || 'phone');
@@ -1961,6 +1966,7 @@ async function detect(){
   const r = await api('/api/detect', {});
   if (r.found){
     st.corners = r.corners; st.measured = r.corner_radius; st.quadFrom = 'detected';
+    orientQuad();
     const corroborated = r.confidence === 'corroborated';
     s.className = 'status ' + (corroborated ? 'ok' : 'warn');
     s.textContent = corroborated ? `Both detectors agree (${r.method})`
@@ -2029,6 +2035,54 @@ function defaultQuad(){
   return [[w*.3,h*.2],[w*.7,h*.2],[w*.7,h*.8],[w*.3,h*.8]];
 }
 $('#redetect').onclick = detect;
+// Rotate: the same four points, the list started one place later. Corner 0 is
+// where the screenshot's top-left goes, so this turns the screenshot a quarter
+// turn clockwise inside an unchanged quad. Needed because detection orders
+// corners by where they sit in the PHOTO (nearest the image's top-left first),
+// and on a phone lying at 45° that is the screen's bottom-left — the quad was
+// right to 1% and the screenshot would have landed sideways (12 Sep 2026).
+// Dragging four corners to their neighbours' places was the only way.
+// The saved fit and the sidecar carry the corners in order, so a rotated fit
+// comes back rotated; nothing else changes.
+function turnQuad(){
+  st.corners = [1, 2, 3, 0].map(i => st.corners[i].slice());
+  if (pick && pick.kind === 'corner') pick = {kind: 'corner', i: (pick.i + 3) % 4};
+  else if (pick && pick.kind === 'edge') pick = {kind: 'edge', i: (pick.i + 3) % 4};
+}
+function rotateQuad(){
+  if (!st.corners) return;
+  turnQuad();
+  draw(); drawStrip(); autoPreview();
+}
+$('#rotatequad').onclick = rotateQuad;
+// The automatic half. Detection orders corners geometrically and cannot know
+// which edge is the top: that depends on the SCREENSHOT. A portrait screenshot
+// goes on the quad the long way, a landscape one the wide way. So once both
+// are known, if the quad's top edge is clearly the wrong kind for the
+// screenshot, turn the order until it is the right kind — the higher of the
+// two candidate top edges, so an upright photo is unchanged. A rule of this
+// shape was first put in detect.py's order_quad and was wrong for every
+// landscape screen; the aspect of the screenshot is the missing fact and only
+// this page has it. Near-square quads (under 1.25) are left alone — perspective
+// can make either kind of edge the longer one. Returns whether it turned.
+const ORIENT_ASPECT = 1.25;
+function orientQuad(){
+  if (!st.corners || !st.shotSize) return false;
+  const C = st.corners;
+  const len = i => Math.hypot(C[(i+1)%4][0]-C[i][0], C[(i+1)%4][1]-C[i][1]);
+  const a = (len(0) + len(2)) / 2, b = (len(1) + len(3)) / 2;   // edges 0/2 vs 1/3
+  if (Math.max(a, b) < ORIENT_ASPECT * Math.max(Math.min(a, b), 1e-6)) return false;
+  const shotPortrait = st.shotSize[1] > st.shotSize[0];
+  const topShouldBeShort = shotPortrait;
+  const topIsShort = a < b;
+  if (topIsShort === topShouldBeShort) return false;
+  // Two candidate top edges (1 and 3, the other pair); the higher one in the
+  // image is the top. Turn once to make edge 1 the top, twice more for edge 3.
+  const midY = i => (C[i][1] + C[(i+1)%4][1]) / 2;
+  const turns = midY(1) <= midY(3) ? 1 : 3;
+  for (let k = 0; k < turns; k++) turnQuad();
+  return true;
+}
 $('#resetquad').onclick = () => {
   if (!img.naturalWidth) return;
   st.corners = defaultQuad(); st.quadFrom = 'default';