screengraft 0.20.4 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/package.json +8 -4
- package/scripts/ui.py +62 -12
- package/scripts/warp.py +50 -10
- package/skills/inject-screenshot/SKILL.md +3 -1
- package/ui/index.html +68 -12
package/README.md
CHANGED
|
@@ -29,6 +29,11 @@ prototype, then put the recording inside a real photograph.
|
|
|
29
29
|
- **Realism pass** *(optional)* — matches the screen's white balance and grain to
|
|
30
30
|
the light in the room, and can lift the device's real reflections from a
|
|
31
31
|
screen-off frame of the same shot.
|
|
32
|
+
- **Emissive screens** *(optional)* — a display emits light *and* reflects the
|
|
33
|
+
room, which is why a switched-off phone looks dark grey rather than black.
|
|
34
|
+
Paint a true-black UI on flat and it reads as a hole cut in the photo. Turn
|
|
35
|
+
this on and the screenshot composites over the device's own glass, so the
|
|
36
|
+
photo's highlights carry across the screen.
|
|
32
37
|
- **Video, not just stills.** The screen source can be an `mp4`/`mov`/`webm`.
|
|
33
38
|
You match the edges on one frame and every frame gets that same geometry — the
|
|
34
39
|
photograph is still, so there is nothing to track and nothing to drift. Output
|
|
@@ -141,6 +146,22 @@ the background is itself neutral (a pale tiled floor, a plain wall) — you plac
|
|
|
141
146
|
the edges by hand there. And a prototype recording has no motion blur, so a very
|
|
142
147
|
fast scroll will strobe; that is a property of the source, not of the composite.
|
|
143
148
|
|
|
149
|
+
## Changelog
|
|
150
|
+
|
|
151
|
+
Every release is described in [CHANGELOG.md](CHANGELOG.md), with the
|
|
152
|
+
measurements that drove it.
|
|
153
|
+
|
|
154
|
+
## Support
|
|
155
|
+
|
|
156
|
+
Bugs and photographs that defeat the detector belong in
|
|
157
|
+
[Issues](https://github.com/seq000/screengraft/issues) — the bug template asks
|
|
158
|
+
first for the `result.json` sidecar, because it reproduces any composite
|
|
159
|
+
exactly. Ideas and "can it do X" go in
|
|
160
|
+
[Discussions](https://github.com/seq000/screengraft/discussions).
|
|
161
|
+
|
|
162
|
+
If the photograph or the UI is confidential — a client shot, something
|
|
163
|
+
unreleased — email **screengraft@fraczyk.design** instead of posting it.
|
|
164
|
+
|
|
144
165
|
## Contributing
|
|
145
166
|
|
|
146
167
|
See [CONTRIBUTING.md](CONTRIBUTING.md). The short version: there are tests, they
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "screengraft",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Put a UI screenshot or screen recording onto a photographed device screen with the perspective exactly right
|
|
3
|
+
"version": "0.21.1",
|
|
4
|
+
"description": "Put a UI screenshot or screen recording onto a photographed device screen with the perspective exactly right \u2014 a homography you confirm by hand, not a generative guess.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mockup",
|
|
7
7
|
"device-frame",
|
|
@@ -17,7 +17,10 @@
|
|
|
17
17
|
"figma"
|
|
18
18
|
],
|
|
19
19
|
"homepage": "https://github.com/seq000/screengraft#readme",
|
|
20
|
-
"bugs":
|
|
20
|
+
"bugs": {
|
|
21
|
+
"url": "https://github.com/seq000/screengraft/issues",
|
|
22
|
+
"email": "screengraft@fraczyk.design"
|
|
23
|
+
},
|
|
21
24
|
"repository": {
|
|
22
25
|
"type": "git",
|
|
23
26
|
"url": "git+https://github.com/seq000/screengraft.git"
|
|
@@ -25,7 +28,8 @@
|
|
|
25
28
|
"license": "MIT",
|
|
26
29
|
"author": {
|
|
27
30
|
"name": "Dariusz Fraczyk",
|
|
28
|
-
"url": "https://fraczyk.design"
|
|
31
|
+
"url": "https://fraczyk.design",
|
|
32
|
+
"email": "screengraft@fraczyk.design"
|
|
29
33
|
},
|
|
30
34
|
"type": "commonjs",
|
|
31
35
|
"bin": {
|
package/scripts/ui.py
CHANGED
|
@@ -183,6 +183,13 @@ def _is_video(path: str) -> bool:
|
|
|
183
183
|
return str(path).lower().endswith(VIDEO_EXT)
|
|
184
184
|
|
|
185
185
|
|
|
186
|
+
def _fit_frame() -> int:
|
|
187
|
+
try:
|
|
188
|
+
return int(SESSION.state.get("fit_frame") or 0)
|
|
189
|
+
except (TypeError, ValueError):
|
|
190
|
+
return 0
|
|
191
|
+
|
|
192
|
+
|
|
186
193
|
def _read_source(path: str):
|
|
187
194
|
"""Read the screen source, which may be a still OR a video.
|
|
188
195
|
|
|
@@ -197,13 +204,27 @@ def _read_source(path: str):
|
|
|
197
204
|
im, rp = _read_image(real)
|
|
198
205
|
return im, rp, {"video": False}
|
|
199
206
|
n, fps, vw, vh = W.probe_video(real)
|
|
200
|
-
frame
|
|
207
|
+
# The frame the designer scrubbed to, NOT frame 0. Preview and Save read the
|
|
208
|
+
# source through here, so reading frame 0 unconditionally made the scrubber
|
|
209
|
+
# look decorative: it moved the thumbnail and the composite never changed
|
|
210
|
+
# (reported 9 Sep 2026). The fitted frame is session state for exactly this
|
|
211
|
+
# reason — more than one route needs it.
|
|
212
|
+
frame = W.read_frame_at(real, _fit_frame())
|
|
201
213
|
# Report the encoder's absence HERE, when the clip is chosen, rather than
|
|
202
214
|
# letting the render fail at the end of the job. Someone who installed
|
|
203
215
|
# screengraft before video existed has a working venv with no ffmpeg in it,
|
|
204
216
|
# and nothing else would tell them until they had done all the fitting.
|
|
217
|
+
# A poster for the chip. The chip used to be handed the .mov path directly,
|
|
218
|
+
# and an <img> cannot render a video, so the thumbnail was silently blank for
|
|
219
|
+
# every clip. It is always FRAME 0 and never follows the scrubber: at chip
|
|
220
|
+
# size one frame looks like any other, so redrawing it would be movement
|
|
221
|
+
# without information.
|
|
222
|
+
poster = os.path.join(SESSION.dir,
|
|
223
|
+
"poster-" + os.path.splitext(os.path.basename(real))[0] + ".jpg")
|
|
224
|
+
if not os.path.exists(poster):
|
|
225
|
+
cv2.imwrite(poster, W.read_frame_at(real, 0), [cv2.IMWRITE_JPEG_QUALITY, 82])
|
|
205
226
|
return frame, real, {"video": True, "frames": n, "fps": fps, "size": [vw, vh],
|
|
206
|
-
"ffmpeg": _have_ffmpeg()}
|
|
227
|
+
"ffmpeg": _have_ffmpeg(), "poster": poster}
|
|
207
228
|
|
|
208
229
|
|
|
209
230
|
# Render progress, read by /api/render_status. A ten-second clip is a few
|
|
@@ -215,14 +236,18 @@ RENDER = {"state": "idle", "done": 0, "total": 0, "output": None, "message": Non
|
|
|
215
236
|
RENDER_LOCK = threading.Lock()
|
|
216
237
|
|
|
217
238
|
|
|
218
|
-
def _render_worker(photo, video_path, corners, dest, radius_px, gr, grain, preset, fit_frame
|
|
239
|
+
def _render_worker(photo, video_path, corners, dest, radius_px, gr, grain, preset, fit_frame,
|
|
240
|
+
blend="replace", reflection=None):
|
|
219
241
|
def progress(done, total):
|
|
220
242
|
with RENDER_LOCK:
|
|
221
243
|
RENDER["done"], RENDER["total"] = done, total
|
|
222
244
|
try:
|
|
223
245
|
info = W.compose_video(photo, video_path, corners, dest,
|
|
224
246
|
corner_radius=radius_px, grade=gr, grain=grain,
|
|
225
|
-
preset=preset, fit_frame=fit_frame, progress=progress
|
|
247
|
+
preset=preset, fit_frame=fit_frame, progress=progress,
|
|
248
|
+
blend=blend,
|
|
249
|
+
reflection=(W.DEFAULT_REFLECTION if reflection is None
|
|
250
|
+
else reflection))
|
|
226
251
|
with RENDER_LOCK:
|
|
227
252
|
RENDER.update(state="done", output=dest, info=info,
|
|
228
253
|
done=info["frames"], total=info["frames"], message=None)
|
|
@@ -231,6 +256,21 @@ def _render_worker(photo, video_path, corners, dest, radius_px, gr, grain, prese
|
|
|
231
256
|
RENDER.update(state="error", message=str(e))
|
|
232
257
|
|
|
233
258
|
|
|
259
|
+
def _blend_args(b):
|
|
260
|
+
"""(blend, reflection) from the page's single `reflection` field.
|
|
261
|
+
|
|
262
|
+
One field, not two: the page sends a number when the switch is on and null
|
|
263
|
+
when it is off, so there is no way to express the contradictory state
|
|
264
|
+
"emissive with no strength" — which is just `replace` under a different
|
|
265
|
+
name. compose() still takes both, because the engine should not have to
|
|
266
|
+
infer intent from a null.
|
|
267
|
+
"""
|
|
268
|
+
r = b.get("reflection")
|
|
269
|
+
if r is None:
|
|
270
|
+
return "replace", W.DEFAULT_REFLECTION
|
|
271
|
+
return "emissive", float(max(0.0, min(1.0, float(r))))
|
|
272
|
+
|
|
273
|
+
|
|
234
274
|
def _guess_type(corners):
|
|
235
275
|
c = np.array(corners, dtype=float)
|
|
236
276
|
w = (np.linalg.norm(c[1] - c[0]) + np.linalg.norm(c[2] - c[3])) / 2
|
|
@@ -345,6 +385,7 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
345
385
|
f.write(self._body())
|
|
346
386
|
# A video is only ever a screen source; a photo must be a still.
|
|
347
387
|
if role == "screenshot":
|
|
388
|
+
SESSION.update(fit_frame=0)
|
|
348
389
|
im, real, meta = _read_source(dest)
|
|
349
390
|
else:
|
|
350
391
|
im, real = _read_image(dest)
|
|
@@ -357,6 +398,7 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
357
398
|
if u.path == "/api/use":
|
|
358
399
|
role = b["role"]
|
|
359
400
|
if role == "screenshot":
|
|
401
|
+
SESSION.update(fit_frame=0)
|
|
360
402
|
im, real, meta = _read_source(b["path"])
|
|
361
403
|
else:
|
|
362
404
|
im, real = _read_image(b["path"])
|
|
@@ -437,12 +479,13 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
437
479
|
spath = _safe_local_path(SESSION.state["screenshot"])
|
|
438
480
|
if not _is_video(spath):
|
|
439
481
|
return self._json({"error": "the screen source is not a video"}, 400)
|
|
482
|
+
# Records which frame the fit is judged on. Nothing is written:
|
|
483
|
+
# the page re-renders the COMPOSITE from it, and the chip stays
|
|
484
|
+
# on frame 0 deliberately, so a per-step PNG would be disk churn
|
|
485
|
+
# nobody looks at.
|
|
440
486
|
idx = int(b.get("index") or 0)
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
cv2.imwrite(dest, frame, [cv2.IMWRITE_PNG_COMPRESSION, 1])
|
|
444
|
-
return self._json({"path": dest, "index": idx,
|
|
445
|
-
"size": [frame.shape[1], frame.shape[0]]})
|
|
487
|
+
SESSION.update(fit_frame=idx)
|
|
488
|
+
return self._json({"index": idx})
|
|
446
489
|
|
|
447
490
|
if u.path == "/api/render":
|
|
448
491
|
# Video: same fit, same geometry, N frames instead of one.
|
|
@@ -460,11 +503,13 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
460
503
|
output=None, message=None)
|
|
461
504
|
corners = b["corners"]
|
|
462
505
|
frac = float(b.get("radius_frac") or 0.0)
|
|
463
|
-
fit_frame = int(b.get("fit_frame")
|
|
506
|
+
fit_frame = int(b.get("fit_frame") if b.get("fit_frame") is not None
|
|
507
|
+
else _fit_frame())
|
|
464
508
|
first = W.read_frame_at(spath, fit_frame)
|
|
465
509
|
radius_px = frac * first.shape[1]
|
|
466
510
|
gr = float(b.get("grade") if b.get("grade") is not None else 0.0)
|
|
467
511
|
grain = bool(b.get("grain", gr > 0))
|
|
512
|
+
blend, reflection = _blend_args(b)
|
|
468
513
|
preset = "prores" if b.get("preset") == "prores" else "web"
|
|
469
514
|
ext = ".mov" if preset == "prores" else ".mp4"
|
|
470
515
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
@@ -484,11 +529,13 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
484
529
|
"corners": corners, "radius_frac": frac, "radius_px": radius_px,
|
|
485
530
|
"device": b.get("device"), "grade": gr, "grain": grain,
|
|
486
531
|
"video": True, "preset": preset, "fit_frame": fit_frame,
|
|
532
|
+
"blend": blend, "reflection": reflection,
|
|
487
533
|
"saved": time.time()}
|
|
488
534
|
_write_json_atomic(SESSION.result_path, result)
|
|
489
535
|
threading.Thread(target=_render_worker, daemon=True,
|
|
490
536
|
args=(photo, spath, corners, dest, radius_px,
|
|
491
|
-
gr, grain, preset, fit_frame
|
|
537
|
+
gr, grain, preset, fit_frame,
|
|
538
|
+
blend, reflection)).start()
|
|
492
539
|
return self._json({"started": True, "output": dest, "preset": preset})
|
|
493
540
|
|
|
494
541
|
if u.path in ("/api/preview", "/api/save"):
|
|
@@ -502,8 +549,10 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
502
549
|
# is the point (a brand review), and the grade is the right one
|
|
503
550
|
# when the photograph is (a portfolio shot).
|
|
504
551
|
gr = float(b.get("grade") if b.get("grade") is not None else 0.0)
|
|
552
|
+
blend, reflection = _blend_args(b)
|
|
505
553
|
out = W.compose(photo, shot, corners, radius_px,
|
|
506
|
-
grade=gr, grain=bool(b.get("grain", gr > 0))
|
|
554
|
+
grade=gr, grain=bool(b.get("grain", gr > 0)),
|
|
555
|
+
blend=blend, reflection=reflection)
|
|
507
556
|
SESSION.update(corners=corners, radius_frac=frac, device=b.get("device"),
|
|
508
557
|
grade=gr)
|
|
509
558
|
if u.path == "/api/preview":
|
|
@@ -537,6 +586,7 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
537
586
|
result = {"output": dest, "photo": ppath, "screenshot": spath, "corners": corners,
|
|
538
587
|
"radius_frac": frac, "radius_px": radius_px, "device": b.get("device"),
|
|
539
588
|
"grade": gr, "grain": bool(b.get("grain", gr > 0)),
|
|
589
|
+
"blend": blend, "reflection": reflection,
|
|
540
590
|
"saved": time.time()}
|
|
541
591
|
_write_json_atomic(SESSION.result_path, result)
|
|
542
592
|
SESSION.update(output=dest)
|
package/scripts/warp.py
CHANGED
|
@@ -40,6 +40,12 @@ import grade as _grade # M2: the realism pass
|
|
|
40
40
|
|
|
41
41
|
MASK_SS = 4 # destination-space supersampling for the screen's edge
|
|
42
42
|
|
|
43
|
+
# How much of the device's own glass shows through under an emissive screen.
|
|
44
|
+
# Measured 9 Sep 2026 on an automotive render whose UI is 56% true black: at 0
|
|
45
|
+
# the screen is a hole, by 25% it sits in the scene, 50% reads clearly as glass,
|
|
46
|
+
# and past 75% the content loses contrast. 0.35 is the middle of the usable band.
|
|
47
|
+
DEFAULT_REFLECTION = 0.35
|
|
48
|
+
|
|
43
49
|
|
|
44
50
|
def rounded_mask(w: int, h: int, radius: float) -> np.ndarray:
|
|
45
51
|
"""White-on-black mask, full frame minus rounded corners cut to black.
|
|
@@ -146,13 +152,16 @@ class Plan:
|
|
|
146
152
|
"""
|
|
147
153
|
|
|
148
154
|
def __init__(self, photo: np.ndarray, frame_shape, corners,
|
|
149
|
-
corner_radius: float = 0.0, grain: bool = False
|
|
155
|
+
corner_radius: float = 0.0, grain: bool = False,
|
|
156
|
+
blend: str = "replace", reflection: float = DEFAULT_REFLECTION):
|
|
150
157
|
dst_quad = np.array(corners, dtype=np.float32)
|
|
151
158
|
if shoelace_area(dst_quad) < 1.0:
|
|
152
159
|
raise ValueError("degenerate quad (near-zero area) — check corner order TL,TR,BR,BL")
|
|
153
160
|
self.photo = photo
|
|
154
161
|
self.dst_quad = dst_quad
|
|
155
162
|
self.grain = grain
|
|
163
|
+
self.blend = blend if blend in ("replace", "emissive") else "replace"
|
|
164
|
+
self.reflection = float(np.clip(reflection, 0.0, 1.0))
|
|
156
165
|
|
|
157
166
|
top = float(np.linalg.norm(dst_quad[1] - dst_quad[0]))
|
|
158
167
|
bottom = float(np.linalg.norm(dst_quad[2] - dst_quad[3]))
|
|
@@ -215,6 +224,31 @@ class Plan:
|
|
|
215
224
|
self.grade_params = _grade.light_params(
|
|
216
225
|
self.photo, self._prep(frame), self.warped_mask, strength) if strength > 0 else None
|
|
217
226
|
|
|
227
|
+
def _blend(self, photo_win, warped_win):
|
|
228
|
+
"""Emitted light over reflected light, or a plain replace.
|
|
229
|
+
|
|
230
|
+
`replace` treats the screenshot as paint: the device's own screen
|
|
231
|
+
surface is discarded. That is right for a reflective surface and wrong
|
|
232
|
+
for an emissive one — a real display shows EMISSION PLUS the room
|
|
233
|
+
reflecting off its glass, which is why a switched-off phone reads dark
|
|
234
|
+
grey and never black. Paint true black onto a lit dashboard and it
|
|
235
|
+
reads as a hole cut in the render (reported 9 Sep 2026 from an
|
|
236
|
+
automotive UI that is 56% #000).
|
|
237
|
+
|
|
238
|
+
`emissive` composites the screenshot OVER the surface instead, with a
|
|
239
|
+
screen blend so highlights cannot blow out, and `reflection` scaling how
|
|
240
|
+
much of the glass survives underneath. The payoff is not only the black
|
|
241
|
+
level: the specular streak running across the dashboard continues
|
|
242
|
+
across the screen, and that continuity is what stops a composite
|
|
243
|
+
reading as an inset panel. No amount of colour-matching can add it,
|
|
244
|
+
because the surface carrying it has already been thrown away.
|
|
245
|
+
"""
|
|
246
|
+
if self.blend != "emissive":
|
|
247
|
+
return warped_win
|
|
248
|
+
P = photo_win.astype(np.float32) * float(self.reflection)
|
|
249
|
+
U = warped_win.astype(np.float32)
|
|
250
|
+
return 255.0 - (255.0 - P) * (255.0 - U) / 255.0
|
|
251
|
+
|
|
218
252
|
def render(self, frame: np.ndarray, screen_off: np.ndarray = None,
|
|
219
253
|
specular: float = 0.75, fast: bool = False) -> np.ndarray:
|
|
220
254
|
"""Composite one frame onto the photo.
|
|
@@ -231,15 +265,16 @@ class Plan:
|
|
|
231
265
|
warped_screen = _grade.apply_light(warped_screen, self.grade_params)
|
|
232
266
|
out = self.photo.copy()
|
|
233
267
|
win = self.mask3[y0:y1, x0:x1]
|
|
268
|
+
pw = self.photo[y0:y1, x0:x1]
|
|
269
|
+
src = self._blend(pw, warped_screen)
|
|
234
270
|
out[y0:y1, x0:x1] = np.clip(
|
|
235
|
-
|
|
236
|
-
+ warped_screen.astype(np.float32) * win, 0, 255).astype(np.uint8)
|
|
271
|
+
pw.astype(np.float32) * (1 - win) + src * win, 0, 255).astype(np.uint8)
|
|
237
272
|
else:
|
|
238
273
|
warped_screen = self._prep(frame)
|
|
239
274
|
if self.grade_params is not None:
|
|
240
275
|
warped_screen = _grade.apply_light(warped_screen, self.grade_params)
|
|
241
|
-
|
|
242
|
-
|
|
276
|
+
src = self._blend(self.photo, warped_screen)
|
|
277
|
+
out = (self.photo.astype(np.float32) * (1 - self.mask3) + src * self.mask3)
|
|
243
278
|
out = np.clip(out, 0, 255).astype(np.uint8)
|
|
244
279
|
if self.grain:
|
|
245
280
|
# Seeded, so the grain is IDENTICAL in every frame. Over a still
|
|
@@ -254,7 +289,8 @@ class Plan:
|
|
|
254
289
|
|
|
255
290
|
def compose(photo: np.ndarray, screenshot: np.ndarray, corners, corner_radius: float = 0.0,
|
|
256
291
|
grade: float = 0.0, grain: bool = False, screen_off: np.ndarray = None,
|
|
257
|
-
specular: float = 0.75
|
|
292
|
+
specular: float = 0.75, blend: str = "replace",
|
|
293
|
+
reflection: float = DEFAULT_REFLECTION) -> np.ndarray:
|
|
258
294
|
"""Warp `screenshot` into the quad `corners` (TL,TR,BR,BL, photo pixels) on `photo`.
|
|
259
295
|
|
|
260
296
|
Single resampling pass at the photo's resolution; deterministic. This is the
|
|
@@ -265,7 +301,8 @@ def compose(photo: np.ndarray, screenshot: np.ndarray, corners, corner_radius: f
|
|
|
265
301
|
# specular) now lives in Plan, so the still and video paths run the SAME
|
|
266
302
|
# code and cannot drift apart. See test_video.py: frame 0 of a render is
|
|
267
303
|
# asserted byte-identical to this function's output.
|
|
268
|
-
plan = Plan(photo, screenshot.shape, corners, corner_radius, grain=grain
|
|
304
|
+
plan = Plan(photo, screenshot.shape, corners, corner_radius, grain=grain,
|
|
305
|
+
blend=blend, reflection=reflection)
|
|
269
306
|
plan.bind_grade(screenshot, grade)
|
|
270
307
|
return plan.render(screenshot, screen_off=screen_off, specular=specular)
|
|
271
308
|
|
|
@@ -349,7 +386,8 @@ def read_frame_at(path: str, index: int = 0):
|
|
|
349
386
|
def compose_video(photo: np.ndarray, video_path: str, corners, output: str,
|
|
350
387
|
corner_radius: float = 0.0, grade: float = 0.0, grain: bool = False,
|
|
351
388
|
preset: str = "web", fit_frame: int = 0, audio: bool = True,
|
|
352
|
-
frames_dir: str = None, progress=None
|
|
389
|
+
frames_dir: str = None, progress=None, blend: str = "replace",
|
|
390
|
+
reflection: float = DEFAULT_REFLECTION) -> dict:
|
|
353
391
|
"""Inject a VIDEO into a still photo. The photo does not move, so there is
|
|
354
392
|
exactly one homography and the whole of Plan is computed once.
|
|
355
393
|
|
|
@@ -367,7 +405,8 @@ def compose_video(photo: np.ndarray, video_path: str, corners, output: str,
|
|
|
367
405
|
"""
|
|
368
406
|
n_hint, fps, vw, vh = probe_video(video_path)
|
|
369
407
|
first = read_frame_at(video_path, fit_frame)
|
|
370
|
-
plan = Plan(photo, first.shape, corners, corner_radius, grain=grain
|
|
408
|
+
plan = Plan(photo, first.shape, corners, corner_radius, grain=grain,
|
|
409
|
+
blend=blend, reflection=reflection)
|
|
371
410
|
plan.bind_grade(first, grade)
|
|
372
411
|
|
|
373
412
|
ph, pw = photo.shape[:2]
|
|
@@ -413,7 +452,8 @@ def compose_video(photo: np.ndarray, video_path: str, corners, output: str,
|
|
|
413
452
|
if count == 0:
|
|
414
453
|
raise RuntimeError(f"no frames could be read from {video_path}")
|
|
415
454
|
return {"frames": count, "fps": fps, "source_size": [vw, vh],
|
|
416
|
-
"output_size": [pw, ph], "preset": preset, "fit_frame": fit_frame
|
|
455
|
+
"output_size": [pw, ph], "preset": preset, "fit_frame": fit_frame,
|
|
456
|
+
"blend": blend, "reflection": plan.reflection}
|
|
417
457
|
|
|
418
458
|
|
|
419
459
|
def main() -> None:
|
|
@@ -5,10 +5,12 @@ description: Injects a UI screenshot OR a screen recording onto a photographed d
|
|
|
5
5
|
|
|
6
6
|
# Inject a screenshot onto a photographed device
|
|
7
7
|
|
|
8
|
-
**What ships (v0.
|
|
8
|
+
**What ships (v0.21):** a local browser UI (`scripts/ui.py`) that walks the designer through the whole job — pick the photo and the screen source, which may be an image **or a video** (recent Desktop/Downloads images, drag-drop, browse, path, or a **Figma frame link**), auto-detect the screen as a starting position, then **match the four edges** (drag an edge's middle to slide it, near an end to pivot; corners still draggable) with canvas zoom/pan and a rectified strip loupe. The fit and the composite sit **side by side and always have** — the result pane re-renders as you drag, which is how a corner gets judged, so it is the layout rather than a mode you can switch off. Then an on-by-default realism pass that colour-matches the source to the photo's light, **Save** (or **Render**, for a video) into the project folder (`--out-dir`), and a **Send to Claude** button that reaches you through the plugin's own MCP server. The UI is a hand port of the project's Figma design file — dark only.
|
|
9
9
|
|
|
10
10
|
The geometry is exact (`warp.py`); the detection is advisory (`detect.py`) and the human corrects it.
|
|
11
11
|
|
|
12
|
+
**Emissive screen** *(off by default)*: a real display shows its own light **plus** the room reflecting off its glass — which is why a switched-off phone reads dark grey and never black. The default `replace` treats the screenshot as paint and discards the device's own screen surface, so a **true-black UI lands as a hole cut in the photo**, and the specular streak running across a dashboard or a phone stops dead at the screen edge. Turn Emissive on and the screenshot is composited *over* the glass instead: blacks become the device's own surface, and the photo's reflections carry across the screen. The strength sets how much glass shows through — **25–50% is the usable band**, past 75% the content loses contrast. Suggest it whenever the user's UI is dark and the composite reads as a flat inset; the realism pass cannot fix that, because it can only lift uniformly and the surface carrying the gradient has already been thrown away.
|
|
13
|
+
|
|
12
14
|
**The realism pass ships and is ON by default** (`grade.py`): it matches the injected screen's white balance and grain to the light around it, at a strength the designer sets in the rail. It can also lift the device's real specular highlights from a screen-off reference frame, though the UI cannot supply one yet. Off is a first-class choice and keeps the screenshot's colour exactly — say so if the user is reviewing brand colour.
|
|
13
15
|
|
|
14
16
|
**Video ships too.** The screen source can be a video (mp4/mov/webm) as well as a still — pick it exactly like a screenshot, choose which frame to match the edges on, and the primary button becomes **Render**. The photo does not move, so there is one homography and every frame gets the same geometry; the light match is measured once from the frame you fitted on, so the screen cannot pulse as the UI scrolls. Output is H.264 at CRF 16 (near-visually-lossless) or ProRes 422 HQ. This is what pairs with a prototype recording: record the prototype, then inject the recording into a real photograph.
|
package/ui/index.html
CHANGED
|
@@ -221,6 +221,12 @@
|
|
|
221
221
|
buttons beside them. Only the corner radii and the shared middle border
|
|
222
222
|
are special. */
|
|
223
223
|
.stepper{display:inline-flex;align-items:center}
|
|
224
|
+
/* A segmented control has to show which segment is on. `.sel` was only ever
|
|
225
|
+
styled for the thumbnail picker (.th.sel), so the web/ProRes pair rendered
|
|
226
|
+
with no selected state at all — reported 9 Sep 2026. Same step as the
|
|
227
|
+
device chips: selected sits on --raise-hi with the brighter border. */
|
|
228
|
+
.stepper button.sel{background:var(--raise-hi);border-color:var(--edge-hi);color:var(--ink)}
|
|
229
|
+
.stepper button.sel:hover{background:var(--raise-highest);border-color:var(--edge-hi)}
|
|
224
230
|
.stepper button{border-radius:0}
|
|
225
231
|
.stepper button:first-child{border-top-left-radius:var(--r-sm);border-bottom-left-radius:var(--r-sm)}
|
|
226
232
|
.stepper button:last-child{border-top-right-radius:var(--r-sm);border-bottom-right-radius:var(--r-sm)}
|
|
@@ -758,8 +764,8 @@
|
|
|
758
764
|
<input type="range" id="vframe" min="0" max="0" value="0" step="1"
|
|
759
765
|
aria-label="Which frame of the clip to match the edges on">
|
|
760
766
|
<span class="sm" id="vframeLbl" style="color:var(--mute);min-width:9ch">0</span>
|
|
761
|
-
<span class="stepper" role="
|
|
762
|
-
<button class="sm sel" data-preset="web" title="H.264 at CRF 16 — near-visually-lossless, plays anywhere.">Web</button><button class="sm" data-preset="prores" title="ProRes 422 HQ — bigger file, no chroma subsampling. For a case study or further editing.">ProRes</button>
|
|
767
|
+
<span class="stepper" role="radiogroup" aria-label="Output format">
|
|
768
|
+
<button class="sm sel" data-preset="web" role="radio" aria-checked="true" title="H.264 at CRF 16 — near-visually-lossless, plays anywhere.">Web</button><button class="sm" data-preset="prores" role="radio" aria-checked="false" title="ProRes 422 HQ — bigger file, no chroma subsampling. For a case study or further editing.">ProRes</button>
|
|
763
769
|
</span>
|
|
764
770
|
</span>
|
|
765
771
|
</div>
|
|
@@ -803,6 +809,23 @@
|
|
|
803
809
|
</div>
|
|
804
810
|
</div></div>
|
|
805
811
|
</div>
|
|
812
|
+
<div class="sect" id="secEmissive" data-on="0">
|
|
813
|
+
<div class="sect-top">
|
|
814
|
+
<div class="sect-left">
|
|
815
|
+
<h4>Emissive screen</h4>
|
|
816
|
+
<button class="info" type="button" aria-label="About emissive screen"><svg viewBox="0 0 16 16" fill="none" aria-hidden="true"><path fill="currentColor" d="M8.36 1.01C12.06 1.2 15 4.26 15 8l-.01.36C14.8 12.06 11.74 15 8 15l-.36-.01C4.06 14.81 1.19 11.94 1.01 8.36L1 8c0-3.87 3.13-7 7-7l.36.01ZM8 2C4.69 2 2 4.69 2 8c0 3.31 2.69 6 6 6 3.31 0 6-2.69 6-6 0-3.31-2.69-6-6-6Zm.5 8l.5 0v1L7 11v-1h.5V7H7V6h1.5v4ZM7.9 4c.33 0 .6.27.6.6a.6.6 0 1 1-1.21 0c0-.33.28-.6.61-.6Z"/></svg><span class="tip" role="tooltip">A real display shows its own light PLUS the room reflecting off its glass — which is why a switched-off phone looks dark grey, never black. Off paints the screenshot on flat, so a true-black UI reads as a hole cut in the photo. On composites it over the glass, so the reflections and highlights already in your photo carry across the screen. Strength sets how much glass shows through.</span></button>
|
|
817
|
+
</div>
|
|
818
|
+
<button class="switch" id="emisBtn" role="switch" aria-checked="false" aria-label="Emissive screen">
|
|
819
|
+
<span class="sw-track"><span class="sw-handle"></span><span class="sw-grip"></span></span>
|
|
820
|
+
</button>
|
|
821
|
+
</div>
|
|
822
|
+
<div class="sect-body"><div class="sect-body-in">
|
|
823
|
+
<div style="display:grid;gap:var(--s2);justify-items:end">
|
|
824
|
+
<span class="sm" id="emisVal" style="font-variant-numeric:tabular-nums"></span>
|
|
825
|
+
<input type="range" id="emisAmt" min="0" max="1" step="0.05" value="0.35" aria-label="Reflection strength">
|
|
826
|
+
</div>
|
|
827
|
+
</div></div>
|
|
828
|
+
</div>
|
|
806
829
|
<div class="sect" id="secEdge" data-on="1">
|
|
807
830
|
<div class="sect-top">
|
|
808
831
|
<div class="sect-left">
|
|
@@ -1130,12 +1153,14 @@ function setChosen(role, path, size, el, meta){
|
|
|
1130
1153
|
const n = role==='photo'?1:2;
|
|
1131
1154
|
document.querySelectorAll('#recent'+n+' .th').forEach(x=>x.classList.remove('sel'));
|
|
1132
1155
|
if (el) el.classList.add('sel');
|
|
1133
|
-
|
|
1156
|
+
// A video cannot be shown by <img>, so a clip gets its poster frame instead.
|
|
1157
|
+
const thumb = (meta && meta.poster) ? meta.poster : path;
|
|
1158
|
+
$('#pv'+n).innerHTML = `<img src="${fileURL(thumb)}" alt="">`;
|
|
1134
1159
|
const chip = $('#chip'+n);
|
|
1135
1160
|
chip.classList.remove('is-empty');
|
|
1136
1161
|
chip.querySelector('.nm').textContent = `${path.split('/').pop()} · ${size[0]}×${size[1]}`
|
|
1137
1162
|
+ (meta && meta.video ? ` · ${meta.frames} frames @ ${Math.round(meta.fps)}fps` : '');
|
|
1138
|
-
$('#sw'+n).style.background = `center/cover no-repeat url("${fileURL(
|
|
1163
|
+
$('#sw'+n).style.background = `center/cover no-repeat url("${fileURL(thumb)}")`;
|
|
1139
1164
|
if (role==='photo'){ st.photo = path; st.corners = null; }
|
|
1140
1165
|
else { st.shot = path; st.video = meta && meta.video ? meta : null; syncVideoUI(); }
|
|
1141
1166
|
closePop();
|
|
@@ -1660,6 +1685,31 @@ $('#gradeBtn').onclick = () => { setGrade(!gradeOn); autoPreview(); };
|
|
|
1660
1685
|
$('#gradeAmt').oninput = e => { gradeAmt = parseFloat(e.target.value); paintGrade(); };
|
|
1661
1686
|
$('#gradeAmt').onchange = e => { setGrade(true, parseFloat(e.target.value)); autoPreview(); };
|
|
1662
1687
|
|
|
1688
|
+
/* Emissive screen. A display emits light AND reflects the room; `replace` models
|
|
1689
|
+
only the first, which is why a true-black UI lands as a hole. Off by default so
|
|
1690
|
+
nothing about an existing fit changes. */
|
|
1691
|
+
let emisOn = recall('emis','0') === '1';
|
|
1692
|
+
let emisAmt = parseFloat(recall('emisAmt','0.35')) || 0.35;
|
|
1693
|
+
function emisValue(){ return emisOn ? emisAmt : null; }
|
|
1694
|
+
function paintEmis(){
|
|
1695
|
+
setSwitch($('#emisBtn'), emisOn);
|
|
1696
|
+
$('#emisVal').textContent = Math.round(emisAmt * 100) + '%';
|
|
1697
|
+
const sl = $('#emisAmt');
|
|
1698
|
+
sl.value = emisAmt;
|
|
1699
|
+
sl.style.setProperty('--p', emisAmt);
|
|
1700
|
+
}
|
|
1701
|
+
function setEmis(on, amt){
|
|
1702
|
+
emisOn = on;
|
|
1703
|
+
if (amt !== undefined) emisAmt = amt;
|
|
1704
|
+
remember('emis', on ? '1' : '0'); remember('emisAmt', String(emisAmt));
|
|
1705
|
+
setSectionOpen($('#secEmissive'), on);
|
|
1706
|
+
paintEmis();
|
|
1707
|
+
}
|
|
1708
|
+
$('#emisBtn').onclick = () => { setEmis(!emisOn); autoPreview(); };
|
|
1709
|
+
$('#emisAmt').oninput = e => { emisAmt = parseFloat(e.target.value); paintEmis(); };
|
|
1710
|
+
$('#emisAmt').onchange = e => { setEmis(true, parseFloat(e.target.value)); autoPreview(); };
|
|
1711
|
+
|
|
1712
|
+
|
|
1663
1713
|
$('#edgeBtn').onclick = () => setLoupeMode(loupeMode === 'float' ? 'dock' : 'float');
|
|
1664
1714
|
|
|
1665
1715
|
function paintStrip(c, W, H, i){
|
|
@@ -1779,7 +1829,7 @@ async function renderPreview(){
|
|
|
1779
1829
|
if (!(st.photo && st.shot && st.corners)) return;
|
|
1780
1830
|
const s = $('#outSt'); s.textContent = 'Rendering…';
|
|
1781
1831
|
try{
|
|
1782
|
-
const r = await api('/api/preview', {corners: st.corners, radius_frac: radiusValue(), device: st.type, grade: gradeValue()});
|
|
1832
|
+
const r = await api('/api/preview', {corners: st.corners, radius_frac: radiusValue(), device: st.type, grade: gradeValue(), reflection: emisValue()});
|
|
1783
1833
|
const im = $('#outImg');
|
|
1784
1834
|
im.onload = () => { outNat = im.naturalWidth; im.hidden = false; $('#outEmpty').style.display='none'; syncOut(); };
|
|
1785
1835
|
im.src = `${fileURL(r.path)}&t=${Date.now()}`;
|
|
@@ -1804,7 +1854,11 @@ let preset = 'web';
|
|
|
1804
1854
|
document.querySelectorAll('[data-preset]').forEach(btn => {
|
|
1805
1855
|
btn.onclick = () => {
|
|
1806
1856
|
preset = btn.dataset.preset;
|
|
1807
|
-
document.querySelectorAll('[data-preset]').forEach(o =>
|
|
1857
|
+
document.querySelectorAll('[data-preset]').forEach(o => {
|
|
1858
|
+
const on = o === btn;
|
|
1859
|
+
o.classList.toggle('sel', on);
|
|
1860
|
+
o.setAttribute('aria-checked', on ? 'true' : 'false');
|
|
1861
|
+
});
|
|
1808
1862
|
};
|
|
1809
1863
|
});
|
|
1810
1864
|
|
|
@@ -1837,10 +1891,11 @@ $('#vframe').onchange = async () => {
|
|
|
1837
1891
|
// Swapping the poster frame re-runs preview through the same path a still
|
|
1838
1892
|
// uses, so what you match the edges against is the frame you chose.
|
|
1839
1893
|
try{
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1894
|
+
await api('/api/frame', {index: +$('#vframe').value});
|
|
1895
|
+
// Only the composite changes. The chip keeps frame 0 on purpose — at that
|
|
1896
|
+
// size one frame looks like any other, so updating it would be movement
|
|
1897
|
+
// without meaning.
|
|
1898
|
+
if (st.corners) renderPreview();
|
|
1844
1899
|
}catch(e){ toast('err', 'Could not read that frame: ' + e.message); }
|
|
1845
1900
|
};
|
|
1846
1901
|
|
|
@@ -1850,7 +1905,7 @@ async function renderVideo(){
|
|
|
1850
1905
|
b.innerHTML = 'Rendering<span class="spin" aria-hidden="true"></span>';
|
|
1851
1906
|
try{
|
|
1852
1907
|
await api('/api/render', {corners: st.corners, radius_frac: radiusValue(), device: st.type,
|
|
1853
|
-
grade: gradeValue(), preset, fit_frame: +$('#vframe').value});
|
|
1908
|
+
grade: gradeValue(), reflection: emisValue(), preset, fit_frame: +$('#vframe').value});
|
|
1854
1909
|
}catch(e){ toast('err','Could not start the render: '+e.message); b.disabled=false; b.textContent='Render'; return; }
|
|
1855
1910
|
// Poll rather than hold a request open: a clip is hundreds of frames and a
|
|
1856
1911
|
// browser would time the request out long before the render finished.
|
|
@@ -1881,7 +1936,7 @@ $('#save').onclick = async () => {
|
|
|
1881
1936
|
if (st.video) return renderVideo();
|
|
1882
1937
|
const b = $('#save'); b.textContent = 'Saving…'; b.disabled = true;
|
|
1883
1938
|
try{
|
|
1884
|
-
const r = await api('/api/save', {corners: st.corners, radius_frac: radiusValue(), device: st.type, grade: gradeValue()});
|
|
1939
|
+
const r = await api('/api/save', {corners: st.corners, radius_frac: radiusValue(), device: st.type, grade: gradeValue(), reflection: emisValue()});
|
|
1885
1940
|
b.textContent = 'Save';
|
|
1886
1941
|
// The real destination, not a hardcoded one: --out-dir means saves usually
|
|
1887
1942
|
// land in the project folder now, and telling the user "~/Desktop" when
|
|
@@ -1941,6 +1996,7 @@ $('#imp').onclick = async () => {
|
|
|
1941
1996
|
}
|
|
1942
1997
|
setLoupeMode(loupeMode);
|
|
1943
1998
|
setGrade(gradeOn, gradeAmt);
|
|
1999
|
+
setEmis(emisOn, emisAmt);
|
|
1944
2000
|
setRadiusOn(radiusOn);
|
|
1945
2001
|
sectionsBooted = true; // animate from here on, not during the first paint
|
|
1946
2002
|
// Compare is the DEFAULT view: judging a fit means comparing it with the
|