screengraft 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +127 -0
- package/bin/screengraft.js +82 -0
- package/mcp/server.py +322 -0
- package/package.json +59 -0
- package/scripts/detect.py +677 -0
- package/scripts/grade.py +180 -0
- package/scripts/launch.sh +69 -0
- package/scripts/preflight.py +118 -0
- package/scripts/requirements.txt +2 -0
- package/scripts/scan.py +92 -0
- package/scripts/stop.sh +21 -0
- package/scripts/ui.py +503 -0
- package/scripts/warp.py +247 -0
- package/skills/inject-screenshot/SKILL.md +130 -0
- package/ui/fonts/OFL.txt +93 -0
- package/ui/fonts/mona-sans-wordmark.woff2 +0 -0
- package/ui/index.html +1902 -0
package/scripts/warp.py
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
screengraft — M0: manual four-point perspective warp.
|
|
4
|
+
|
|
5
|
+
Takes a photo of a device and a UI screenshot, and warps the screenshot into
|
|
6
|
+
a given quadrilateral (the four screen corners in the photo) so the
|
|
7
|
+
perspective matches exactly. Geometry, not generation: a single
|
|
8
|
+
cv2.getPerspectiveTransform + cv2.warpPerspective call, deterministic
|
|
9
|
+
end to end.
|
|
10
|
+
|
|
11
|
+
Design constraints this script exists to satisfy (see the project's design notes):
|
|
12
|
+
- Warp once, at the photo's full resolution — never warp-then-scale
|
|
13
|
+
(double resampling blurs text).
|
|
14
|
+
- Fully deterministic: same inputs -> byte-identical output PNG.
|
|
15
|
+
- No AI, no guessing: corners are supplied by the caller (a human, via the
|
|
16
|
+
drag-picker UI, or hardcoded for a test) — this script only does the
|
|
17
|
+
textbook part.
|
|
18
|
+
|
|
19
|
+
Usage:
|
|
20
|
+
python3 warp.py --photo photo.jpg --screenshot ui.png \
|
|
21
|
+
--corners '[[120,80],[860,140],[840,900],[100,840]]' \
|
|
22
|
+
--output out.png [--corner-radius 40]
|
|
23
|
+
|
|
24
|
+
Corners are [x, y] pixel coordinates in the PHOTO, in order
|
|
25
|
+
TL, TR, BR, BL (top-left, top-right, bottom-right, bottom-left of the
|
|
26
|
+
screen as it appears in the photo — order matters, it defines the mapping).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
import argparse
|
|
30
|
+
import json
|
|
31
|
+
import sys
|
|
32
|
+
|
|
33
|
+
import cv2
|
|
34
|
+
import numpy as np
|
|
35
|
+
|
|
36
|
+
import grade as _grade # M2: the realism pass
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
MASK_SS = 4 # destination-space supersampling for the screen's edge
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def rounded_mask(w: int, h: int, radius: float) -> np.ndarray:
|
|
43
|
+
"""White-on-black mask, full frame minus rounded corners cut to black.
|
|
44
|
+
|
|
45
|
+
Computed analytically rather than drawn, for two reasons.
|
|
46
|
+
|
|
47
|
+
Geometry. The screenshot occupies the edge-coordinate box [0,w]x[0,h] --
|
|
48
|
+
the box `compose` hands to getPerspectiveTransform -- so pixel *centres*
|
|
49
|
+
sit at i+0.5 and the four arc centres belong at (r, r) and (w-r, h-r) in
|
|
50
|
+
that same edge space. The previous version drew the arcs with cv2.circle,
|
|
51
|
+
whose integer coordinates are pixel *centres*, which placed the whole
|
|
52
|
+
rounded rectangle half a pixel down and to the right. The top and left
|
|
53
|
+
arcs still came out tangent to the frame, but the bottom and right arcs
|
|
54
|
+
were tangent to y=h and x=w -- one row/column outside the image -- so each
|
|
55
|
+
was clipped a pixel early and met its straight edge at a slope
|
|
56
|
+
discontinuity instead of flattening into it. Reported from a real save, 4 Sep 2026
|
|
57
|
+
as the bottom corners looking "cut out by a pixel or two"; the row
|
|
58
|
+
coverage profile confirmed it, the bottom being exactly the top shifted
|
|
59
|
+
by one row (the tangent row missing entirely).
|
|
60
|
+
|
|
61
|
+
Antialiasing. Coverage is a 1px linear ramp on the distance to the
|
|
62
|
+
rounded rectangle, symmetric on all four sides by construction. LINE_AA's
|
|
63
|
+
own ramp is not, and cannot be nudged sub-pixel without the fixed-point
|
|
64
|
+
`shift` dance.
|
|
65
|
+
"""
|
|
66
|
+
if radius <= 0:
|
|
67
|
+
return np.full((h, w), 255, dtype=np.uint8)
|
|
68
|
+
r = float(max(0.0, min(float(radius), w / 2.0, h / 2.0)))
|
|
69
|
+
xs = np.arange(w, dtype=np.float64) + 0.5 # pixel centres, edge coords
|
|
70
|
+
ys = np.arange(h, dtype=np.float64) + 0.5
|
|
71
|
+
# Per-axis distance past the arc-centre rail: zero everywhere except the
|
|
72
|
+
# four corner squares, so `dist` is the true distance to the rounded
|
|
73
|
+
# rectangle's boundary there and the straight edges stay exactly full.
|
|
74
|
+
dx = np.maximum(np.maximum(r - xs, xs - (w - r)), 0.0)
|
|
75
|
+
dy = np.maximum(np.maximum(r - ys, ys - (h - r)), 0.0)
|
|
76
|
+
dist = np.hypot(dx[None, :], dy[:, None])
|
|
77
|
+
coverage = np.clip(r + 0.5 - dist, 0.0, 1.0)
|
|
78
|
+
return np.rint(coverage * 255.0).astype(np.uint8)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _warp_mask_antialiased(src_mask, H, pw, ph, dst_quad):
|
|
82
|
+
"""Warp the screen mask into photo space with coverage antialiasing.
|
|
83
|
+
|
|
84
|
+
A single warpPerspective of a binary mask gives a ~1px hard transition,
|
|
85
|
+
so the screen's edge lands on the photo stepped — obvious on a slanted
|
|
86
|
+
edge, which is every interesting photo (reported 4 Sep 2026).
|
|
87
|
+
|
|
88
|
+
Instead the mask is rasterised at MASK_SS x the output scale and then
|
|
89
|
+
INTER_AREA'd down, so each output pixel gets the *fraction* of itself the
|
|
90
|
+
screen actually covers — the same thing multisampling does. Only the
|
|
91
|
+
quad's bounding box is supersampled, so the cost is a few megapixels
|
|
92
|
+
rather than MASK_SS^2 x the whole photo.
|
|
93
|
+
"""
|
|
94
|
+
x0 = max(0, int(np.floor(dst_quad[:, 0].min())) - 2)
|
|
95
|
+
y0 = max(0, int(np.floor(dst_quad[:, 1].min())) - 2)
|
|
96
|
+
x1 = min(pw, int(np.ceil(dst_quad[:, 0].max())) + 2)
|
|
97
|
+
y1 = min(ph, int(np.ceil(dst_quad[:, 1].max())) + 2)
|
|
98
|
+
out = np.zeros((ph, pw), dtype=np.uint8)
|
|
99
|
+
if x1 <= x0 or y1 <= y0:
|
|
100
|
+
return out
|
|
101
|
+
bw, bh = x1 - x0, y1 - y0
|
|
102
|
+
# photo space -> supersampled bbox-local space
|
|
103
|
+
T = np.array([[MASK_SS, 0, -MASK_SS * x0],
|
|
104
|
+
[0, MASK_SS, -MASK_SS * y0],
|
|
105
|
+
[0, 0, 1]], dtype=np.float64)
|
|
106
|
+
big = cv2.warpPerspective(
|
|
107
|
+
src_mask, T @ H, (bw * MASK_SS, bh * MASK_SS),
|
|
108
|
+
flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)
|
|
109
|
+
out[y0:y1, x0:x1] = cv2.resize(big, (bw, bh), interpolation=cv2.INTER_AREA)
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def shoelace_area(pts: np.ndarray) -> float:
|
|
114
|
+
x = pts[:, 0]
|
|
115
|
+
y = pts[:, 1]
|
|
116
|
+
return 0.5 * abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1)))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def compose(photo: np.ndarray, screenshot: np.ndarray, corners, corner_radius: float = 0.0,
|
|
120
|
+
grade: float = 0.0, grain: bool = False, screen_off: np.ndarray = None,
|
|
121
|
+
specular: float = 0.75) -> np.ndarray:
|
|
122
|
+
"""Warp `screenshot` into the quad `corners` (TL,TR,BR,BL, photo pixels) on `photo`.
|
|
123
|
+
|
|
124
|
+
Single resampling pass at the photo's resolution; deterministic. This is the
|
|
125
|
+
whole engine — the CLI below and ui.py both call it.
|
|
126
|
+
"""
|
|
127
|
+
dst_quad = np.array(corners, dtype=np.float32)
|
|
128
|
+
if shoelace_area(dst_quad) < 1.0:
|
|
129
|
+
raise ValueError("degenerate quad (near-zero area) — check corner order TL,TR,BR,BL")
|
|
130
|
+
|
|
131
|
+
# --- prefilter for minification -------------------------------------
|
|
132
|
+
# A UI screenshot is almost always far larger than the screen it lands
|
|
133
|
+
# on: 1206x2622 into a 226x454 quad is 5.3x across and 5.8x along, so
|
|
134
|
+
# each output pixel is the average of ~31 source pixels. warpPerspective
|
|
135
|
+
# (like remap) does NOT area-average — INTER_LANCZOS4 samples a fixed 8x8
|
|
136
|
+
# window around one source point no matter the scale, and Lanczos is a
|
|
137
|
+
# sharpening kernel, so heavy minification came out aliased and crunchy
|
|
138
|
+
# with text turned to noise (3 Sep 2026, reported from a real save).
|
|
139
|
+
#
|
|
140
|
+
# The fix is the mipmap principle: area-average DOWN to roughly the
|
|
141
|
+
# destination footprint first, then warp at ~1:1. This is not the
|
|
142
|
+
# "warp-then-scale" the build brief warns against — that's resampling an
|
|
143
|
+
# already-warped result, which blurs. This is resampling the source with
|
|
144
|
+
# the right filter before the only geometric pass, which is how you avoid
|
|
145
|
+
# aliasing when minifying.
|
|
146
|
+
top = float(np.linalg.norm(dst_quad[1] - dst_quad[0]))
|
|
147
|
+
bottom = float(np.linalg.norm(dst_quad[2] - dst_quad[3]))
|
|
148
|
+
left = float(np.linalg.norm(dst_quad[3] - dst_quad[0]))
|
|
149
|
+
right = float(np.linalg.norm(dst_quad[2] - dst_quad[1]))
|
|
150
|
+
# Use the LONGER opposing edge of each pair: under perspective the near
|
|
151
|
+
# edge carries the most detail, and that's the resolution to preserve.
|
|
152
|
+
need_w = max(top, bottom)
|
|
153
|
+
need_h = max(left, right)
|
|
154
|
+
sh0, sw0 = screenshot.shape[:2]
|
|
155
|
+
scale_x, scale_y = need_w / sw0, need_h / sh0
|
|
156
|
+
if 0 < max(scale_x, scale_y) < 0.95: # only ever downsample
|
|
157
|
+
new_w = max(1, int(round(sw0 * max(scale_x, scale_y))))
|
|
158
|
+
new_h = max(1, int(round(sh0 * max(scale_x, scale_y))))
|
|
159
|
+
screenshot = cv2.resize(screenshot, (new_w, new_h), interpolation=cv2.INTER_AREA)
|
|
160
|
+
corner_radius = corner_radius * (new_w / sw0) # radius is in source px
|
|
161
|
+
# --------------------------------------------------------------------
|
|
162
|
+
|
|
163
|
+
sh, sw = screenshot.shape[:2]
|
|
164
|
+
src_rect = np.array([[0, 0], [sw, 0], [sw, sh], [0, sh]], dtype=np.float32)
|
|
165
|
+
H = cv2.getPerspectiveTransform(src_rect, dst_quad)
|
|
166
|
+
ph, pw = photo.shape[:2]
|
|
167
|
+
# Radius stays fractional: rounded_mask is analytic, and after the
|
|
168
|
+
# prefilter rescale above a truncation here is up to a whole pixel of
|
|
169
|
+
# radius thrown away at exactly the scale the viewer is looking at.
|
|
170
|
+
src_mask = rounded_mask(sw, sh, float(corner_radius))
|
|
171
|
+
# BORDER_REPLICATE, not BORDER_CONSTANT black: with an antialiased mask
|
|
172
|
+
# the edge pixels are a genuine blend of screen and photo, and sampling
|
|
173
|
+
# black just outside the screenshot would draw a dark fringe right where
|
|
174
|
+
# the antialiasing is supposed to be doing its work. Replicating the edge
|
|
175
|
+
# pixel means a half-covered pixel blends real screen colour instead.
|
|
176
|
+
warped_screen = cv2.warpPerspective(
|
|
177
|
+
screenshot, H, (pw, ph), flags=cv2.INTER_LANCZOS4,
|
|
178
|
+
borderMode=cv2.BORDER_REPLICATE)
|
|
179
|
+
warped_mask = _warp_mask_antialiased(src_mask, H, pw, ph, dst_quad)
|
|
180
|
+
|
|
181
|
+
# --- M2: the realism pass -------------------------------------------
|
|
182
|
+
# Order is not arbitrary. The light match runs on the warped screen BEFORE
|
|
183
|
+
# compositing, so it measures and moves only screen pixels — grading after
|
|
184
|
+
# the blend would drag the bezel with it. Grain and the specular lift run
|
|
185
|
+
# AFTER, because both are things that happen to the finished surface, and
|
|
186
|
+
# both are confined to the screen by the same mask.
|
|
187
|
+
if grade > 0:
|
|
188
|
+
warped_screen = _grade.match_light(photo, warped_screen, warped_mask, strength=grade)
|
|
189
|
+
|
|
190
|
+
mask3 = cv2.merge([warped_mask] * 3).astype(np.float32) / 255.0
|
|
191
|
+
out = photo.astype(np.float32) * (1 - mask3) + warped_screen.astype(np.float32) * mask3
|
|
192
|
+
out = np.clip(out, 0, 255).astype(np.uint8)
|
|
193
|
+
|
|
194
|
+
if grain:
|
|
195
|
+
sigma = _grade.measure_grain(photo, _grade.surround_ring(warped_mask))
|
|
196
|
+
out = _grade.add_grain(out, warped_mask, sigma)
|
|
197
|
+
if screen_off is not None:
|
|
198
|
+
out = _grade.specular_lift(out, screen_off, warped_mask, strength=specular)
|
|
199
|
+
return out
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def main() -> None:
|
|
203
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
204
|
+
ap.add_argument("--photo", required=True, help="Path to the device photo")
|
|
205
|
+
ap.add_argument("--screenshot", required=True, help="Path to the UI screenshot to inject")
|
|
206
|
+
ap.add_argument("--corners", required=True, help="JSON [[x,y]x4] in the photo, order TL,TR,BR,BL")
|
|
207
|
+
ap.add_argument("--output", required=True, help="Output PNG path")
|
|
208
|
+
ap.add_argument("--corner-radius", type=float, default=0.0,
|
|
209
|
+
help="Corner radius in SCREENSHOT source pixels (0 = square corners, M0 default)")
|
|
210
|
+
args = ap.parse_args()
|
|
211
|
+
|
|
212
|
+
photo = cv2.imread(args.photo, cv2.IMREAD_COLOR)
|
|
213
|
+
if photo is None:
|
|
214
|
+
sys.exit(f"error: could not read photo: {args.photo}")
|
|
215
|
+
screenshot = cv2.imread(args.screenshot, cv2.IMREAD_COLOR)
|
|
216
|
+
if screenshot is None:
|
|
217
|
+
sys.exit(f"error: could not read screenshot: {args.screenshot}")
|
|
218
|
+
|
|
219
|
+
try:
|
|
220
|
+
corners = json.loads(args.corners)
|
|
221
|
+
except json.JSONDecodeError as e:
|
|
222
|
+
sys.exit(f"error: --corners is not valid JSON: {e}")
|
|
223
|
+
if len(corners) != 4 or any(len(c) != 2 for c in corners):
|
|
224
|
+
sys.exit("error: --corners must be a JSON list of exactly 4 [x, y] pairs")
|
|
225
|
+
|
|
226
|
+
try:
|
|
227
|
+
composite = compose(photo, screenshot, corners, args.corner_radius)
|
|
228
|
+
except ValueError as e:
|
|
229
|
+
sys.exit(f"error: {e}")
|
|
230
|
+
sh, sw = screenshot.shape[:2]
|
|
231
|
+
ph, pw = photo.shape[:2]
|
|
232
|
+
|
|
233
|
+
ok = cv2.imwrite(args.output, composite, [cv2.IMWRITE_PNG_COMPRESSION, 9])
|
|
234
|
+
if not ok:
|
|
235
|
+
sys.exit(f"error: could not write output: {args.output}")
|
|
236
|
+
|
|
237
|
+
print(json.dumps({
|
|
238
|
+
"output": args.output,
|
|
239
|
+
"photo_size": [pw, ph],
|
|
240
|
+
"screenshot_size": [sw, sh],
|
|
241
|
+
"dst_quad": corners,
|
|
242
|
+
"corner_radius": args.corner_radius,
|
|
243
|
+
}))
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
if __name__ == "__main__":
|
|
247
|
+
main()
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: inject-screenshot
|
|
3
|
+
description: Injects a UI screenshot onto a photographed device screen (phone/tablet/laptop) at any angle, matching the perspective exactly via homography — geometry, not AI generation. Use when the user wants to composite a screen design into a real device photo for a portfolio, case study, or mockup, and needs the result to look like a genuine photo rather than a template. Opens a local browser UI for picking files, correcting corners and saving.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Inject a screenshot onto a photographed device
|
|
7
|
+
|
|
8
|
+
**What ships (v0.13.1):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screenshot (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position, then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas zoom/pan and a rectified strip loupe, Preview (in a popup, or continuously in the compare pane), an on-by-default realism pass that colour-matches the screenshot to the photo's light, Save into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
|
|
9
|
+
|
|
10
|
+
The geometry is exact (`warp.py`); the detection is advisory (`detect.py`) and the human corrects it.
|
|
11
|
+
|
|
12
|
+
**The realism pass ships and is ON by default** (`grade.py`, M2): it matches the injected screen's white balance and grain to the light around it, at a strength the designer sets in the rail. It can also lift the device's real specular highlights from a screen-off reference frame, though the UI cannot supply one yet. Off is a first-class choice and keeps the screenshot's colour exactly — say so if the user is reviewing brand colour.
|
|
13
|
+
|
|
14
|
+
Still missing: **no ML detection** (M4 — measured, and it segments the phone body rather than the glass, so it is not shipped), **no occluder handling** — a finger or glare in front of the screen gets painted over (M5) — and **no video** (M3). Say so if it matters for the photo.
|
|
15
|
+
|
|
16
|
+
**Runs on the user's Mac shell** (Desktop Commander `start_process` or equivalent). The sandboxed Linux shell can't open a browser or reach `~/Desktop`. Paths below are relative to the plugin root — two levels up from this file.
|
|
17
|
+
|
|
18
|
+
## Workflow
|
|
19
|
+
|
|
20
|
+
### 0. Preflight — every time, before promising anything
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
python3 scripts/preflight.py
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Read the JSON. If `ready` is true, note `python` — **use that interpreter for every command below** (it's the venv at `~/.screengraft/venv`, not system Python). If `ready` is false:
|
|
27
|
+
|
|
28
|
+
- Tell the user plainly what's missing and that **OpenCV is the engine: without it nothing runs — not worse results, no results.**
|
|
29
|
+
- Ask with `AskUserQuestion` whether to run the install: `python3 scripts/preflight.py --install` (creates `~/.screengraft/venv`, pip-installs `opencv-python-headless` + `numpy`, ~60 MB, touches nothing else). Run it only after a yes. This is the **only** interview question this skill asks in chat — everything else happens in the UI.
|
|
30
|
+
|
|
31
|
+
### 1. Launch the UI — with the project folder as the output directory
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
scripts/launch.sh --out-dir "<the user's current project folder>/mockups"
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
**Always pass `--out-dir`** when you know the folder the user is working in. Two reasons, and the second is not optional: saves land where the designer actually is rather than on the Desktop, and `present_files` **refuses any path outside a connected folder** — so with the default Desktop output, the "Import to Claude" button cannot work at all. If you genuinely have no project folder, omit the flag and tell the user that Import will be unavailable.
|
|
38
|
+
|
|
39
|
+
**Do not launch `ui.py` directly with `nohup ... &` from your own shell.** That was tried on 3 Sep 2026 and the server died silently between turns, twice, losing whatever the user had already entered — the shell session that launches it can be torn down out from under a merely-backgrounded child. `launch.sh` starts it with `--daemon`, which double-forks and calls `setsid`, putting the server in its own session with no controlling terminal; it survives the launching shell and opens no window. It self-checks (waits for startup output, then confirms the server responds) before returning.
|
|
40
|
+
|
|
41
|
+
A Terminal.app window was the previous fix for the same problem. It worked but left a dead "[Process completed]" window behind after every session, and closing those from AppleScript proved unreliable — Terminal reports stale ttys for dead windows and leaves zero-tab husks that `close` reports success on. The daemon removes the window entirely, so there is nothing to clean up.
|
|
42
|
+
|
|
43
|
+
It prints one JSON line — `url`, `session`, `job`, `result`, `out_dir` — and opens the browser tab itself. If it exits non-zero, read the error and the log path it names before telling the user anything is ready.
|
|
44
|
+
|
|
45
|
+
**Verify liveness again right before telling the user to interact with it** — `curl -sf <url>api/state` — especially if any time has passed since launch. If that fails, the server died; say so, relaunch, and have the user redo their last action rather than assuming it's still there.
|
|
46
|
+
|
|
47
|
+
**Then explain the job in chat.** The page deliberately carries no onboarding — the explanation belongs here, where the user already is. Say this, in your own words but keeping all of it:
|
|
48
|
+
|
|
49
|
+
> **What this does** — it computes the perspective between your photo and your screenshot, so the screenshot lands on the glass exactly. Geometry, not AI: nothing is invented and your pixels are unchanged.
|
|
50
|
+
>
|
|
51
|
+
> **1 · Choose a photo, then a screenshot.** Recent images from Desktop and Downloads are listed for you — or drag a file in, browse, paste a path, or paste a Figma frame link and I'll export it.
|
|
52
|
+
>
|
|
53
|
+
> **2 · Match the four edges to the screen.** Drag an edge's middle to slide it, or near an end to pivot — only that edge moves. The magnified strip below shows the boundary straightened, so aligned reads as flat. Arrow keys nudge 1px, Shift+arrow 10px, Tab moves to the next edge.
|
|
54
|
+
>
|
|
55
|
+
> **3 · Preview, then Save**, then **Send to Claude** and I'll check the result and show it here. Saves go to `<out-dir>`.
|
|
56
|
+
>
|
|
57
|
+
> A detector proposes a starting quad, but it's only a guess — you confirm all four edges. That's deliberate: a confident-looking wrong result is the one failure this tool won't risk.
|
|
58
|
+
|
|
59
|
+
Adapt it: name the real output folder, and mention the realism pass only if it matters (it is on by default and changes the screenshot's colour, which is worth flagging if they are reviewing brand colour). Say it once, on launch — not again on every re-arm.
|
|
60
|
+
|
|
61
|
+
Do not build a chat *interview* — the page collects every input, and duplicating its questions is what the one-interview-surface rule forbids. Explaining the workflow is not an interview. The page scans `~/Desktop` and `~/Downloads` for recent images itself, every launch.
|
|
62
|
+
|
|
63
|
+
### 2. Park in `wait_for_job` — this is the main loop, not an optional extra
|
|
64
|
+
|
|
65
|
+
**You cannot poll.** You run in turns; between them nothing of you is executing. An earlier version of this file told you to "poll `job.json` every few seconds" and the page told the user "(it watches this job)" — both false, and the user sat waiting for an agent that was not running. Do not reproduce that in chat: never tell the user you are watching, monitoring, or keeping an eye on anything.
|
|
66
|
+
|
|
67
|
+
What you *can* do is block. Immediately after launching the UI, call:
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
wait_for_job(timeout_s=90)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
It parks until the user presses a button and returns within ~150 ms of the press. Handle what comes back, then **call it again**. That loop is how the page reaches you.
|
|
74
|
+
|
|
75
|
+
| result | what to do |
|
|
76
|
+
|---|---|
|
|
77
|
+
| `status=job`, `job.type="figma_export"` | Extract `fileKey` and `nodeId` from `job.url` (`1-2` → `1:2`), export via the Figma MCP (`download_assets`, PNG, scale 3), save to `job.save_to`, then `complete_job(status="done", path=<saved file>)`. Re-arm. |
|
|
78
|
+
| `status=job`, `job.type="present"` | `present_files` on `job.paths`, report what you checked in the composite (§3), then `complete_job(status="done")`. Re-arm. |
|
|
79
|
+
| `status=timeout` | Nothing pressed yet. Call again to keep waiting. Re-arm two or three times, then stop and say you've stopped waiting — do not loop forever burning turns. |
|
|
80
|
+
| `status=ui_closed` | The UI exited. Stop; say so. |
|
|
81
|
+
| `status=no_ui` | Nothing is running — launch it first. |
|
|
82
|
+
|
|
83
|
+
Pass the job's `id` back as `job_id` when you complete it. If the user gave up and started something else while you were working, that stops your late answer from marking their new request done and stranding them on a spinner.
|
|
84
|
+
|
|
85
|
+
**Always `complete_job`, including on failure.** The page is polling for it and will sit on a spinner until it arrives. If the Figma MCP isn't available, `complete_job(status="error", message="…")` with a sentence the designer can act on — the page shows it and offers the manual route.
|
|
86
|
+
|
|
87
|
+
While you are blocked you cannot do anything else, which is correct during a fit but means the user's own chat messages wait for the call to return. That is why the default is 90 s rather than the 150 s maximum: it bounds how long a message can sit behind the wait.
|
|
88
|
+
|
|
89
|
+
**Don't raise `timeout_s` past the default hoping to park longer.** The host kills an MCP tool call at around 180 s — measured on 4 Sep 2026, when a `wait_for_job(300)` was killed at exactly 180 s. Nothing is lost when that happens (an unanswered job stays pending and the next wait picks it up), but you miss the wake you were parked for, which is the whole point. The server clamps to 150 s; re-arming is the way to wait longer, not a bigger number.
|
|
90
|
+
|
|
91
|
+
### 3. When `<session>/result.json` appears: verify, then hand over
|
|
92
|
+
|
|
93
|
+
The user pressed Save. Read the output image back (you can see images). Check:
|
|
94
|
+
|
|
95
|
+
- The injected screen sits on the bezel edge all the way round — no sliver of the original screen showing, no UI poking past the glass. Zoom a corner if unsure.
|
|
96
|
+
- Text in the injected area is sharp. Soft means double resampling — that's a bug, not a setting.
|
|
97
|
+
- Nothing that was in front of the screen in the photo has been painted over (if it has, say so — M5).
|
|
98
|
+
|
|
99
|
+
Then `present_files` the output. Report what you checked, not "done".
|
|
100
|
+
|
|
101
|
+
If the user is unhappy, the UI is still open — they nudge and Save again; a new file is written (`-2`, `-3`, …), never overwritten.
|
|
102
|
+
|
|
103
|
+
### 4. Finish
|
|
104
|
+
|
|
105
|
+
The server is a detached daemon with no window. **Stop it with `scripts/stop.sh`, never a bare `pkill -9`.** stop.sh sends SIGTERM, which ui.py catches so its atexit clears the session pointer the MCP server reads; a hard kill leaves that pointer stale and the next `wait_for_job` then blocks a full timeout against a session nobody is in. Sessions live in `~/.screengraft/sessions/<timestamp>/` — leave them, they're small.
|
|
106
|
+
|
|
107
|
+
## Rules
|
|
108
|
+
|
|
109
|
+
- **Preflight first, install only on a yes.** OpenCV is required, full stop — never describe its absence as "reduced quality".
|
|
110
|
+
- **One interview surface, but explain the job in chat.** The browser page collects every *input*; `AskUserQuestion` is for install consent only, and you must not re-ask in chat what the page already asks. Explaining what the tool does and what the three steps are is not an interview — it is the onboarding, it belongs in chat, and §1 gives the words.
|
|
111
|
+
- **Never claim a composite is right without reading the output back.** Correct-looking code is not evidence; neither is the user clicking Save.
|
|
112
|
+
- **Detection is advisory and is only a starting position; the warp is exact — keep the two claims apart.** No-ML detection was measured to its ceiling (see the design notes): it cannot tell a screen from a bezel from a body, because that is a semantic question. The edge-matching UI exists so a human fixes it in seconds, and the page shows green ONLY when both detectors corroborate each other.
|
|
113
|
+
- **Testing the plugin ≠ doing the task.** If the user is testing screengraft and it can't do something, say what it can't do and propose the change. Don't reach past the plugin with ad-hoc scripts to make the result better.
|
|
114
|
+
- **Never claim to be watching, monitoring, or waiting on something you aren't.** You have exactly one way to be reached from the page — being blocked inside `wait_for_job`. If you are not in that call, you are not reachable, and saying otherwise leaves the user waiting on nothing. This is the specific failure v0.7 was built to fix; don't reintroduce it in prose.
|
|
115
|
+
|
|
116
|
+
## Scripts
|
|
117
|
+
|
|
118
|
+
| File | What it does |
|
|
119
|
+
|---|---|
|
|
120
|
+
| `mcp/server.py` | The plugin's MCP server: `wait_for_job` / `complete_job`. Stdlib only on system `python3`, deliberately — it must load before preflight has built the venv, or the tools would silently be missing on a fresh install. |
|
|
121
|
+
| `scripts/preflight.py` | Dependency report; `--install` builds the venv |
|
|
122
|
+
| `scripts/launch.sh` | Starts `ui.py` as a detached daemon (no window) and verifies it's alive — use this, not `ui.py` directly |
|
|
123
|
+
| `scripts/stop.sh` | Stops the UI with SIGTERM so the session pointer is cleared — use this instead of `pkill` |
|
|
124
|
+
| `scripts/ui.py` | Local server + browser UI; calls the two below |
|
|
125
|
+
| `scripts/detect.py` | Advisory screen-quad + corner-radius detection (no ML) |
|
|
126
|
+
| `scripts/warp.py` | The engine: `compose()` and a CLI for scripted use |
|
|
127
|
+
| `scripts/scan.py` | Recent images on Desktop/Downloads, thumbnails |
|
|
128
|
+
| `ui/index.html` | The page |
|
|
129
|
+
|
|
130
|
+
Design rationale and the roadmap live with the project's own notes, not in the plugin.
|
package/ui/fonts/OFL.txt
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Copyright 2022 The Mona Sans Project Authors (https://github.com/github/mona-sans), with Reserved Font Name "Mona"
|
|
2
|
+
|
|
3
|
+
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
|
4
|
+
This license is copied below, and is also available with a FAQ at:
|
|
5
|
+
https://openfontlicense.org
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
-----------------------------------------------------------
|
|
9
|
+
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
|
10
|
+
-----------------------------------------------------------
|
|
11
|
+
|
|
12
|
+
PREAMBLE
|
|
13
|
+
The goals of the Open Font License (OFL) are to stimulate worldwide
|
|
14
|
+
development of collaborative font projects, to support the font creation
|
|
15
|
+
efforts of academic and linguistic communities, and to provide a free and
|
|
16
|
+
open framework in which fonts may be shared and improved in partnership
|
|
17
|
+
with others.
|
|
18
|
+
|
|
19
|
+
The OFL allows the licensed fonts to be used, studied, modified and
|
|
20
|
+
redistributed freely as long as they are not sold by themselves. The
|
|
21
|
+
fonts, including any derivative works, can be bundled, embedded,
|
|
22
|
+
redistributed and/or sold with any software provided that any reserved
|
|
23
|
+
names are not used by derivative works. The fonts and derivatives,
|
|
24
|
+
however, cannot be released under any other type of license. The
|
|
25
|
+
requirement for fonts to remain under this license does not apply
|
|
26
|
+
to any document created using the fonts or their derivatives.
|
|
27
|
+
|
|
28
|
+
DEFINITIONS
|
|
29
|
+
"Font Software" refers to the set of files released by the Copyright
|
|
30
|
+
Holder(s) under this license and clearly marked as such. This may
|
|
31
|
+
include source files, build scripts and documentation.
|
|
32
|
+
|
|
33
|
+
"Reserved Font Name" refers to any names specified as such after the
|
|
34
|
+
copyright statement(s).
|
|
35
|
+
|
|
36
|
+
"Original Version" refers to the collection of Font Software components as
|
|
37
|
+
distributed by the Copyright Holder(s).
|
|
38
|
+
|
|
39
|
+
"Modified Version" refers to any derivative made by adding to, deleting,
|
|
40
|
+
or substituting -- in part or in whole -- any of the components of the
|
|
41
|
+
Original Version, by changing formats or by porting the Font Software to a
|
|
42
|
+
new environment.
|
|
43
|
+
|
|
44
|
+
"Author" refers to any designer, engineer, programmer, technical
|
|
45
|
+
writer or other person who contributed to the Font Software.
|
|
46
|
+
|
|
47
|
+
PERMISSION & CONDITIONS
|
|
48
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
49
|
+
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
|
50
|
+
redistribute, and sell modified and unmodified copies of the Font
|
|
51
|
+
Software, subject to the following conditions:
|
|
52
|
+
|
|
53
|
+
1) Neither the Font Software nor any of its individual components,
|
|
54
|
+
in Original or Modified Versions, may be sold by itself.
|
|
55
|
+
|
|
56
|
+
2) Original or Modified Versions of the Font Software may be bundled,
|
|
57
|
+
redistributed and/or sold with any software, provided that each copy
|
|
58
|
+
contains the above copyright notice and this license. These can be
|
|
59
|
+
included either as stand-alone text files, human-readable headers or
|
|
60
|
+
in the appropriate machine-readable metadata fields within text or
|
|
61
|
+
binary files as long as those fields can be easily viewed by the user.
|
|
62
|
+
|
|
63
|
+
3) No Modified Version of the Font Software may use the Reserved Font
|
|
64
|
+
Name(s) unless explicit written permission is granted by the corresponding
|
|
65
|
+
Copyright Holder. This restriction only applies to the primary font name as
|
|
66
|
+
presented to the users.
|
|
67
|
+
|
|
68
|
+
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
|
69
|
+
Software shall not be used to promote, endorse or advertise any
|
|
70
|
+
Modified Version, except to acknowledge the contribution(s) of the
|
|
71
|
+
Copyright Holder(s) and the Author(s) or with their explicit written
|
|
72
|
+
permission.
|
|
73
|
+
|
|
74
|
+
5) The Font Software, modified or unmodified, in part or in whole,
|
|
75
|
+
must be distributed entirely under this license, and must not be
|
|
76
|
+
distributed under any other license. The requirement for fonts to
|
|
77
|
+
remain under this license does not apply to any document created
|
|
78
|
+
using the Font Software.
|
|
79
|
+
|
|
80
|
+
TERMINATION
|
|
81
|
+
This license becomes null and void if any of the above conditions are
|
|
82
|
+
not met.
|
|
83
|
+
|
|
84
|
+
DISCLAIMER
|
|
85
|
+
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
86
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
|
87
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
|
88
|
+
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
|
89
|
+
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
90
|
+
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
|
91
|
+
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
92
|
+
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
|
93
|
+
OTHER DEALINGS IN THE FONT SOFTWARE.
|
|
Binary file
|