screengraft 0.25.2 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -0
- package/package.json +1 -1
- package/scripts/detect.py +335 -36
- package/scripts/fitfile.py +140 -0
- package/scripts/fits.py +151 -0
- package/scripts/ui.py +398 -42
- package/scripts/warp.py +155 -7
- package/skills/inject-screenshot/SKILL.md +3 -3
- package/ui/index.html +723 -67
package/scripts/ui.py
CHANGED
|
@@ -47,6 +47,8 @@ sys.path.insert(0, HERE)
|
|
|
47
47
|
import cv2 # noqa: E402
|
|
48
48
|
import numpy as np # noqa: E402
|
|
49
49
|
import detect as D # noqa: E402
|
|
50
|
+
import fitfile as FF # noqa: E402
|
|
51
|
+
import fits as FIT # noqa: E402
|
|
50
52
|
import scan as S # noqa: E402
|
|
51
53
|
import warp as W # noqa: E402
|
|
52
54
|
|
|
@@ -76,17 +78,104 @@ def _write_json_atomic(path, obj):
|
|
|
76
78
|
os.replace(tmp, path)
|
|
77
79
|
|
|
78
80
|
# Corner radius as a fraction of the SCREEN'S WIDTH, per device preset.
|
|
79
|
-
#
|
|
80
|
-
#
|
|
81
|
-
#
|
|
81
|
+
#
|
|
82
|
+
# The iPhone values are DERIVED, not estimated: radius in points from Apple's
|
|
83
|
+
# private `UIScreen._displayCornerRadius`, as collected by kylebshr/ScreenCorners,
|
|
84
|
+
# divided by the model's logical width in points. Both are exact, so the fraction
|
|
85
|
+
# is too — the only approximation left is that a photographed screen is not
|
|
86
|
+
# always exactly its logical aspect.
|
|
87
|
+
#
|
|
88
|
+
# There is no single "iPhone radius" and the spread is large: 39pt on an iPhone X
|
|
89
|
+
# against 62pt on a 17 Pro, and because width also changes, two models with the
|
|
90
|
+
# SAME radius land on different fractions (55pt is 14.0% of a 393pt iPhone 16 and
|
|
91
|
+
# 12.8% of a 430pt 16 Plus). A single preset could not have covered these.
|
|
92
|
+
#
|
|
93
|
+
# group radius width frac
|
|
94
|
+
# iPhone 17 Pro / 17 / 16 Pro 62.0 402 0.154
|
|
95
|
+
# iPhone Air 62.0 420 0.148
|
|
96
|
+
# iPhone 17 Pro Max / 16 Pro Max 62.0 440 0.141
|
|
97
|
+
# iPhone 16 / 15 Pro / 15 / 14 Pro 55.0 393 0.140
|
|
98
|
+
# iPhone 16 Plus / 15 Pro Max / 15 Plus /
|
|
99
|
+
# 14 Pro Max 55.0 430 0.128
|
|
100
|
+
# iPhone 14 Plus / 13 Pro Max / 12 Pro Max 53.33 428 0.125
|
|
101
|
+
# iPhone 16e / 14 / 13 Pro / 13 / 12 Pro / 12 47.33 390 0.121
|
|
102
|
+
# iPhone 13 mini / 12 mini 44.0 375 0.117
|
|
103
|
+
# iPhone 11 / XR 41.5 414 0.100
|
|
104
|
+
# iPhone 11 Pro / XS / X 39.0 375 0.104
|
|
105
|
+
# iPhone 11 Pro Max / XS Max 39.0 414 0.094
|
|
106
|
+
# iPhone SE / 8 / 7 0 - 0
|
|
107
|
+
#
|
|
108
|
+
# Two caveats worth knowing before trusting a preset over a measurement:
|
|
109
|
+
#
|
|
110
|
+
# 1. ScreenCorners lists the plain **iPhone 13** nowhere, while listing the 13
|
|
111
|
+
# Pro at 47.33. The 13 and 13 Pro share a 390x844 display, so 47.33 is the
|
|
112
|
+
# inference here rather than a reported value. It is the only entry below
|
|
113
|
+
# that is not directly attested.
|
|
114
|
+
# 2. Apple's display corners are a **continuous curve**, not a circular arc, and
|
|
115
|
+
# compose() applies a circular radius. Matched by number they are not matched
|
|
116
|
+
# by shape — a continuous corner reads slightly tighter at the diagonal. The
|
|
117
|
+
# preset is a starting position; the measured radius, when the photograph
|
|
118
|
+
# gives one, beats it.
|
|
119
|
+
#
|
|
120
|
+
# `label` is what fits the closed dropdown at its real width. `full` is the
|
|
121
|
+
# exact membership and the numbers it came from, and the page puts it in the
|
|
122
|
+
# CAPTION under the slider — not in a tooltip on the <option>, which a native
|
|
123
|
+
# macOS select popup does not reliably render. The short label must never be
|
|
124
|
+
# the only place the truth lives.
|
|
125
|
+
#
|
|
126
|
+
# `smoothing` is Figma's corner smoothing for that device: Apple's display
|
|
127
|
+
# corners are a continuous curve, not a circular arc, so every Apple preset
|
|
128
|
+
# carries 0.6 — the value Figma labels "iOS". Android and the square entries
|
|
129
|
+
# carry nothing and stay circular. It is a property of the DEVICE, which is why
|
|
130
|
+
# it lives here rather than in a control: a photograph of an iPhone does not
|
|
131
|
+
# have a smoothing preference, it has a shape.
|
|
132
|
+
#
|
|
133
|
+
# iPads: 18pt on every rounded model, over 744pt (mini) / 834pt (11" / Air) /
|
|
134
|
+
# 1024pt (12.9"). MacBook display corners ~12px on ~1500pt. Monitors square.
|
|
82
135
|
PRESETS = [
|
|
83
|
-
|
|
84
|
-
{"id": "phone-iphone-
|
|
136
|
+
# Newest first — that is the order a photograph is likely to be of.
|
|
137
|
+
{"id": "phone-iphone-17pro", "type": "phone", "smoothing": 0.6, "frac": 0.154,
|
|
138
|
+
"label": "iPhone 17 Pro / 17 / 16 Pro",
|
|
139
|
+
"full": "iPhone 17 Pro, iPhone 17, iPhone 16 Pro \u2014 62pt over 402pt"},
|
|
140
|
+
{"id": "phone-iphone-air", "type": "phone", "smoothing": 0.6, "frac": 0.148,
|
|
141
|
+
"label": "iPhone Air",
|
|
142
|
+
"full": "iPhone Air \u2014 62pt over 420pt"},
|
|
143
|
+
{"id": "phone-iphone-17promax", "type": "phone", "smoothing": 0.6, "frac": 0.141,
|
|
144
|
+
"label": "iPhone 17 Pro Max / 16 Pro Max",
|
|
145
|
+
"full": "iPhone 17 Pro Max, iPhone 16 Pro Max \u2014 62pt over 440pt"},
|
|
146
|
+
# Keeps its original id: this group is what "iPhone 15 / 16 / Pro" meant.
|
|
147
|
+
{"id": "phone-iphone", "type": "phone", "smoothing": 0.6, "frac": 0.140,
|
|
148
|
+
"label": "iPhone 16 / 15 / 15 Pro / 14 Pro",
|
|
149
|
+
"full": "iPhone 16, 15, 15 Pro, 14 Pro \u2014 55pt over 393pt"},
|
|
150
|
+
{"id": "phone-iphone-max", "type": "phone", "smoothing": 0.6, "frac": 0.128,
|
|
151
|
+
"label": "iPhone 15\u201316 Plus, 14\u201315 Pro Max",
|
|
152
|
+
"full": "iPhone 16 Plus, 15 Plus, 15 Pro Max, 14 Pro Max \u2014 55pt over 430pt"},
|
|
153
|
+
{"id": "phone-iphone-14plus", "type": "phone", "smoothing": 0.6, "frac": 0.125,
|
|
154
|
+
"label": "iPhone 14 Plus / 12\u201313 Pro Max",
|
|
155
|
+
"full": "iPhone 14 Plus, 13 Pro Max, 12 Pro Max \u2014 53.33pt over 428pt"},
|
|
156
|
+
{"id": "phone-iphone-12", "type": "phone", "smoothing": 0.6, "frac": 0.121,
|
|
157
|
+
"label": "iPhone 12\u201314 / 12\u201313 Pro / 16e",
|
|
158
|
+
"full": "iPhone 14, 13, 13 Pro, 12, 12 Pro, 16e \u2014 47.33pt over 390pt. The plain 13 is inferred: it shares the 390pt display with the 13 Pro."},
|
|
159
|
+
{"id": "phone-iphone-mini", "type": "phone", "smoothing": 0.6, "frac": 0.117,
|
|
160
|
+
"label": "iPhone 13 mini / 12 mini",
|
|
161
|
+
"full": "iPhone 13 mini, 12 mini \u2014 44pt over 375pt"},
|
|
162
|
+
{"id": "phone-iphone-x", "type": "phone", "smoothing": 0.6, "frac": 0.104,
|
|
163
|
+
"label": "iPhone 11 Pro / XS / X",
|
|
164
|
+
"full": "iPhone 11 Pro, XS, X \u2014 39pt over 375pt"},
|
|
165
|
+
{"id": "phone-iphone-xr", "type": "phone", "smoothing": 0.6, "frac": 0.100,
|
|
166
|
+
"label": "iPhone 11 / XR",
|
|
167
|
+
"full": "iPhone 11, XR \u2014 41.5pt over 414pt"},
|
|
168
|
+
{"id": "phone-iphone-xsmax", "type": "phone", "smoothing": 0.6, "frac": 0.094,
|
|
169
|
+
"label": "iPhone 11 Pro Max / XS Max",
|
|
170
|
+
"full": "iPhone 11 Pro Max, XS Max \u2014 39pt over 414pt"},
|
|
171
|
+
{"id": "phone-iphone-se", "type": "phone", "frac": 0.0,
|
|
172
|
+
"label": "iPhone SE / 8 / 7",
|
|
173
|
+
"full": "iPhone SE (2nd/3rd gen), 8, 7 \u2014 square display corners, no radius"},
|
|
85
174
|
{"id": "phone-android", "type": "phone", "label": "Android (typical)", "frac": 0.090},
|
|
86
|
-
{"id": "tablet-ipad-pro-11", "type": "tablet", "label": "iPad Pro 11 / Air", "frac": 0.022},
|
|
87
|
-
{"id": "tablet-ipad-pro-13", "type": "tablet", "label": "iPad Pro 13", "frac": 0.018},
|
|
88
|
-
{"id": "tablet-ipad-mini", "type": "tablet", "label": "iPad mini", "frac": 0.024},
|
|
89
|
-
{"id": "laptop-macbook", "type": "laptop", "label": "MacBook Air / Pro", "frac": 0.008},
|
|
175
|
+
{"id": "tablet-ipad-pro-11", "type": "tablet", "smoothing": 0.6, "label": "iPad Pro 11 / Air", "frac": 0.022},
|
|
176
|
+
{"id": "tablet-ipad-pro-13", "type": "tablet", "smoothing": 0.6, "label": "iPad Pro 13", "frac": 0.018},
|
|
177
|
+
{"id": "tablet-ipad-mini", "type": "tablet", "smoothing": 0.6, "label": "iPad mini", "frac": 0.024},
|
|
178
|
+
{"id": "laptop-macbook", "type": "laptop", "smoothing": 0.6, "label": "MacBook Air / Pro", "frac": 0.008},
|
|
90
179
|
{"id": "laptop-other", "type": "laptop", "label": "Other laptop (square)", "frac": 0.0},
|
|
91
180
|
{"id": "desktop", "type": "desktop", "label": "Desktop monitor (square)", "frac": 0.0},
|
|
92
181
|
]
|
|
@@ -152,7 +241,7 @@ SESSION: Session = None
|
|
|
152
241
|
_RESIDUE_PREFIXES = ("photo-", "screenshot-", "poster-", "frame-")
|
|
153
242
|
# figma-export.png is a copy too -- the agent fetches the frame and drops it
|
|
154
243
|
# here -- and re-exporting is one MCP round trip, so it is residue like the rest.
|
|
155
|
-
_RESIDUE_NAMES = ("preview.png", "figma-export.png")
|
|
244
|
+
_RESIDUE_NAMES = ("preview.png", "preview.mp4", "figma-export.png")
|
|
156
245
|
|
|
157
246
|
|
|
158
247
|
def _sidecar_sources(d):
|
|
@@ -467,12 +556,28 @@ def _read_source(path: str):
|
|
|
467
556
|
# an HTTP request open — so the render runs on its own thread and the page
|
|
468
557
|
# polls. ThreadingHTTPServer is already the server class, so this needs no
|
|
469
558
|
# other machinery.
|
|
470
|
-
RENDER = {"state": "idle", "done": 0, "total": 0, "output": None, "message": None
|
|
559
|
+
RENDER = {"state": "idle", "done": 0, "total": 0, "output": None, "message": None,
|
|
560
|
+
"kind": "render"}
|
|
471
561
|
RENDER_LOCK = threading.Lock()
|
|
472
562
|
|
|
473
563
|
|
|
564
|
+
# How wide a preview proxy is composited. Small enough that a clip can be
|
|
565
|
+
# watched in seconds rather than minutes, large enough that the two things a
|
|
566
|
+
# preview exists to judge -- the light match holding across the clip, and the
|
|
567
|
+
# emissive blend against changing content -- are visible. It is a PROXY: it
|
|
568
|
+
# never reaches --out-dir and never becomes the session output.
|
|
569
|
+
PREVIEW_WIDTH = 720
|
|
570
|
+
# ...and how much of the clip. A 2460-frame recording takes about as long to
|
|
571
|
+
# composite as the render it is meant to save you from, and a preview you wait a
|
|
572
|
+
# minute for is a render with a worse output. Six seconds from the frame you
|
|
573
|
+
# fitted on is enough to watch the light match hold and the emissive blend move
|
|
574
|
+
# with the content; the scrubber picks a different moment if you want one.
|
|
575
|
+
PREVIEW_SECONDS = 6.0
|
|
576
|
+
|
|
577
|
+
|
|
474
578
|
def _render_worker(photo, video_path, corners, dest, radius_px, gr, grain, preset, fit_frame,
|
|
475
|
-
blend="replace", reflection=None, result=None
|
|
579
|
+
blend="replace", reflection=None, result=None, kind="render",
|
|
580
|
+
start_frame=0, max_frames=None, *, smoothing=0.0):
|
|
476
581
|
"""Encode the clip, and only if that SUCCEEDS publish what it produced.
|
|
477
582
|
|
|
478
583
|
`result` is the sidecar this render would write. It is handed to the worker
|
|
@@ -491,24 +596,57 @@ def _render_worker(photo, video_path, corners, dest, radius_px, gr, grain, prese
|
|
|
491
596
|
RENDER["done"], RENDER["total"] = done, total
|
|
492
597
|
try:
|
|
493
598
|
info = W.compose_video(photo, video_path, corners, dest,
|
|
494
|
-
corner_radius=radius_px,
|
|
599
|
+
corner_radius=radius_px, corner_smoothing=smoothing,
|
|
600
|
+
grade=gr, grain=grain,
|
|
495
601
|
preset=preset, fit_frame=fit_frame, progress=progress,
|
|
496
602
|
blend=blend,
|
|
497
603
|
reflection=(W.DEFAULT_REFLECTION if reflection is None
|
|
498
|
-
else reflection)
|
|
604
|
+
else reflection),
|
|
605
|
+
start_frame=start_frame, max_frames=max_frames)
|
|
606
|
+
if kind == "preview":
|
|
607
|
+
# A preview publishes NOTHING. It is not a save: no sidecar, no fit
|
|
608
|
+
# file, and above all not the session output -- /api/import reads
|
|
609
|
+
# that pointer, and handing Claude a 720px proxy instead of the
|
|
610
|
+
# mockup would be the render-output defect of 9 Sep, inverted.
|
|
611
|
+
with RENDER_LOCK:
|
|
612
|
+
RENDER.update(state="done", output=dest, info=info, kind=kind,
|
|
613
|
+
done=info["frames"], total=info["frames"], message=None)
|
|
614
|
+
return
|
|
499
615
|
if result is not None:
|
|
500
616
|
_write_json_atomic(SESSION.result_path, {**result, "saved": time.time()})
|
|
617
|
+
# Same rule as /api/save, and for the same reason it lives after the
|
|
618
|
+
# encode: the fit is remembered by the run that produced a file.
|
|
619
|
+
key = FIT.key_for(photo)
|
|
620
|
+
FIT.remember(key, corners, result.get("radius_frac"),
|
|
621
|
+
result.get("device"), result.get("photo"),
|
|
622
|
+
(photo.shape[1], photo.shape[0]))
|
|
623
|
+
FF.write(dest, FF.build(corners, result.get("radius_frac"),
|
|
624
|
+
result.get("device"), result.get("photo"),
|
|
625
|
+
(photo.shape[1], photo.shape[0]), key))
|
|
501
626
|
# The still path has always done this (see /api/save); the render path
|
|
502
627
|
# never did, so /api/import -- which reads SESSION.state["output"] --
|
|
503
628
|
# either found nothing or, worse, silently handed over the PREVIOUS
|
|
504
629
|
# still image after a successful render.
|
|
505
630
|
SESSION.update(output=dest)
|
|
506
631
|
with RENDER_LOCK:
|
|
507
|
-
RENDER.update(state="done", output=dest, info=info,
|
|
632
|
+
RENDER.update(state="done", output=dest, info=info, kind=kind,
|
|
508
633
|
done=info["frames"], total=info["frames"], message=None)
|
|
509
634
|
except Exception as e: # noqa: BLE001 - surfaced to the page
|
|
510
635
|
with RENDER_LOCK:
|
|
511
|
-
RENDER.update(state="error", message=str(e))
|
|
636
|
+
RENDER.update(state="error", message=str(e), kind=kind)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _smoothing(b) -> float:
|
|
640
|
+
"""Figma corner smoothing for this request, 0-1.
|
|
641
|
+
|
|
642
|
+
Absent means 0, which is a circular arc — so a client that predates
|
|
643
|
+
smoothing, and a sidecar replayed through the CLI, both get the shape they
|
|
644
|
+
got before. That default is the compatibility guarantee, not a convenience.
|
|
645
|
+
"""
|
|
646
|
+
try:
|
|
647
|
+
return float(min(max(float(b.get("smoothing") or 0.0), 0.0), 1.0))
|
|
648
|
+
except (TypeError, ValueError):
|
|
649
|
+
return 0.0
|
|
512
650
|
|
|
513
651
|
|
|
514
652
|
def _blend_args(b):
|
|
@@ -526,6 +664,44 @@ def _blend_args(b):
|
|
|
526
664
|
return "emissive", float(max(0.0, min(1.0, float(r))))
|
|
527
665
|
|
|
528
666
|
|
|
667
|
+
ROLES = ("photo", "screenshot")
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _adopt(role, path):
|
|
671
|
+
"""Make a chosen source the session's, and answer what the page needs.
|
|
672
|
+
|
|
673
|
+
/api/use and /api/upload differ only in where the bytes came from --
|
|
674
|
+
everything after that (the video probe, the session state, and now the
|
|
675
|
+
remembered fit) has to be identical for a drag-drop and a path pick, or the
|
|
676
|
+
feature works one way in and not the other. It was already written twice;
|
|
677
|
+
the fit lookup would have made it three, and a set of writers that
|
|
678
|
+
disagree is a defect this project has already shipped once.
|
|
679
|
+
"""
|
|
680
|
+
# The role names a session-state key, and the line below writes it, so an
|
|
681
|
+
# unchecked role is a write primitive: role="output" sets the pointer
|
|
682
|
+
# /api/import reads, and the agent is then asked to show whatever file that
|
|
683
|
+
# names. Nothing on this port authenticates, so the caller is not
|
|
684
|
+
# necessarily the page. Two roles exist; anything else is a 400.
|
|
685
|
+
if role not in ROLES:
|
|
686
|
+
raise ValueError(f"role must be one of {', '.join(ROLES)}")
|
|
687
|
+
if role == "screenshot":
|
|
688
|
+
SESSION.update(fit_frame=0)
|
|
689
|
+
im, real, meta = _read_source(path)
|
|
690
|
+
else:
|
|
691
|
+
im, real = _read_image(path)
|
|
692
|
+
meta = {"video": False}
|
|
693
|
+
SESSION.update(**{role: real})
|
|
694
|
+
if role == "photo":
|
|
695
|
+
# The quad is a property of the PHOTOGRAPH, so a fit saved on an earlier
|
|
696
|
+
# run is a better starting position than any detector -- and a stronger
|
|
697
|
+
# claim, which is why the page states where the quad came from rather
|
|
698
|
+
# than presenting a memory as a detection.
|
|
699
|
+
e = FIT.recall(FIT.key_for(im))
|
|
700
|
+
if e:
|
|
701
|
+
meta["remembered"] = e
|
|
702
|
+
return {"path": real, "size": [im.shape[1], im.shape[0]], **meta}
|
|
703
|
+
|
|
704
|
+
|
|
529
705
|
def _guess_type(corners):
|
|
530
706
|
c = np.array(corners, dtype=float)
|
|
531
707
|
w = (np.linalg.norm(c[1] - c[0]) + np.linalg.norm(c[2] - c[3])) / 2
|
|
@@ -558,13 +734,54 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
558
734
|
self.wfile.write(body)
|
|
559
735
|
|
|
560
736
|
def _file(self, path, ctype=None):
|
|
737
|
+
"""Serve a local file, answering a Range request when one is made.
|
|
738
|
+
|
|
739
|
+
A browser cannot SEEK in a video the server will only hand over whole:
|
|
740
|
+
it plays from the start and every jump snaps back to zero. That is not a
|
|
741
|
+
detail here -- the live view exists to be parked on the frame the edges
|
|
742
|
+
were matched against, and the frame scrubber is supposed to move it.
|
|
743
|
+
Measured before fixing: setting currentTime to 9.0s read back as 0.0.
|
|
744
|
+
|
|
745
|
+
So: advertise `Accept-Ranges`, and answer a single `bytes=a-b` with a
|
|
746
|
+
206. Multi-range is not implemented and is not needed -- media players
|
|
747
|
+
ask for one range at a time -- and anything unparseable falls through to
|
|
748
|
+
the whole file, which is what the spec asks for.
|
|
749
|
+
"""
|
|
561
750
|
try:
|
|
562
|
-
|
|
563
|
-
|
|
751
|
+
size = os.path.getsize(path)
|
|
752
|
+
f = open(path, "rb")
|
|
564
753
|
except OSError:
|
|
565
754
|
return self._json({"error": "not found"}, 404)
|
|
566
|
-
|
|
567
|
-
|
|
755
|
+
ctype = ctype or mimetypes.guess_type(path)[0] or "application/octet-stream"
|
|
756
|
+
start, end = 0, size - 1
|
|
757
|
+
partial = False
|
|
758
|
+
rng = self.headers.get("Range") or ""
|
|
759
|
+
if rng.startswith("bytes=") and "," not in rng:
|
|
760
|
+
a, _, b = rng[6:].partition("-")
|
|
761
|
+
try:
|
|
762
|
+
if a:
|
|
763
|
+
start = int(a)
|
|
764
|
+
end = int(b) if b else size - 1
|
|
765
|
+
elif b: # bytes=-N: the LAST n bytes
|
|
766
|
+
start = max(0, size - int(b))
|
|
767
|
+
if 0 <= start <= end < size:
|
|
768
|
+
partial = True
|
|
769
|
+
else:
|
|
770
|
+
start, end = 0, size - 1
|
|
771
|
+
except ValueError:
|
|
772
|
+
start, end = 0, size - 1
|
|
773
|
+
try:
|
|
774
|
+
with f:
|
|
775
|
+
if partial:
|
|
776
|
+
f.seek(start)
|
|
777
|
+
data = f.read(end - start + 1) if partial else f.read()
|
|
778
|
+
except OSError:
|
|
779
|
+
return self._json({"error": "not readable"}, 404)
|
|
780
|
+
self.send_response(206 if partial else 200)
|
|
781
|
+
self.send_header("Content-Type", ctype)
|
|
782
|
+
self.send_header("Accept-Ranges", "bytes")
|
|
783
|
+
if partial:
|
|
784
|
+
self.send_header("Content-Range", f"bytes {start}-{end}/{size}")
|
|
568
785
|
self.send_header("Content-Length", str(len(data)))
|
|
569
786
|
self.send_header("Cache-Control", "no-store")
|
|
570
787
|
self.end_headers()
|
|
@@ -648,27 +865,12 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
648
865
|
with open(dest, "wb") as f:
|
|
649
866
|
f.write(self._body())
|
|
650
867
|
# A video is only ever a screen source; a photo must be a still.
|
|
651
|
-
|
|
652
|
-
SESSION.update(fit_frame=0)
|
|
653
|
-
im, real, meta = _read_source(dest)
|
|
654
|
-
else:
|
|
655
|
-
im, real = _read_image(dest)
|
|
656
|
-
meta = {"video": False}
|
|
657
|
-
SESSION.update(**{role: real})
|
|
658
|
-
return self._json({"path": real, "size": [im.shape[1], im.shape[0]], **meta})
|
|
868
|
+
return self._json(_adopt(role, dest))
|
|
659
869
|
|
|
660
870
|
b = self._jbody()
|
|
661
871
|
|
|
662
872
|
if u.path == "/api/use":
|
|
663
|
-
|
|
664
|
-
if role == "screenshot":
|
|
665
|
-
SESSION.update(fit_frame=0)
|
|
666
|
-
im, real, meta = _read_source(b["path"])
|
|
667
|
-
else:
|
|
668
|
-
im, real = _read_image(b["path"])
|
|
669
|
-
meta = {"video": False}
|
|
670
|
-
SESSION.update(**{role: real})
|
|
671
|
-
return self._json({"path": real, "size": [im.shape[1], im.shape[0]], **meta})
|
|
873
|
+
return self._json(_adopt(b["role"], b["path"]))
|
|
672
874
|
|
|
673
875
|
if u.path == "/api/figma":
|
|
674
876
|
return self._json(SESSION.enqueue({
|
|
@@ -703,6 +905,29 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
703
905
|
SESSION.update(screenshot=real)
|
|
704
906
|
return self._json({"path": real, "size": [im.shape[1], im.shape[0]]})
|
|
705
907
|
|
|
908
|
+
if u.path == "/api/fit":
|
|
909
|
+
# A fit file dropped onto the page. The page reads the bytes and
|
|
910
|
+
# posts the parsed document; only the server can judge it,
|
|
911
|
+
# because judging it means knowing what the OPEN photograph is.
|
|
912
|
+
#
|
|
913
|
+
# It never applies silently. Corners are meaningless on the wrong
|
|
914
|
+
# image and *plausible but wrong* on a crop of the right one,
|
|
915
|
+
# which is the more dangerous of the two — so every answer says
|
|
916
|
+
# which of four situations it is and the page says it out loud.
|
|
917
|
+
if not SESSION.state.get("photo"):
|
|
918
|
+
raise ValueError("choose the photo first, then drop the fit onto it")
|
|
919
|
+
doc, err = FF.parse(b.get("fit"))
|
|
920
|
+
if err:
|
|
921
|
+
return self._json({"error": err}, 400)
|
|
922
|
+
photo, _p = _read_image(SESSION.state["photo"])
|
|
923
|
+
corners, match, message = FF.apply_to(
|
|
924
|
+
doc, (photo.shape[1], photo.shape[0]), FIT.key_for(photo))
|
|
925
|
+
return self._json({"corners": corners, "match": match,
|
|
926
|
+
"message": message,
|
|
927
|
+
"radius_frac": doc.get("radius_frac"),
|
|
928
|
+
"device": doc.get("device"),
|
|
929
|
+
"from": doc.get("photo", {}).get("name")})
|
|
930
|
+
|
|
706
931
|
if u.path == "/api/detect":
|
|
707
932
|
if not SESSION.state.get("photo"):
|
|
708
933
|
raise ValueError("choose a photo first")
|
|
@@ -725,9 +950,23 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
725
950
|
raise ValueError("the click is outside the photograph")
|
|
726
951
|
# `color` gives detect() the saturation detector — devices are
|
|
727
952
|
# neutral, furniture is not, and grayscale throws that away.
|
|
728
|
-
|
|
953
|
+
# The instrument, off unless asked for: the full
|
|
954
|
+
# candidate list, with each quad's score and what became of it,
|
|
955
|
+
# written beside the session. The page never shows it -- it is
|
|
956
|
+
# for the next person diagnosing a detection failure, and for a
|
|
957
|
+
# benchmark that needs to tell "never proposed" from "proposed
|
|
958
|
+
# and beaten". The response carries the counts and the path, not
|
|
959
|
+
# several hundred quads nobody asked the page to render.
|
|
960
|
+
trace = [] if (isinstance(b, dict) and b.get("trace")) else None
|
|
961
|
+
res = D.detect(gray, None, color=photo, click=click, trace=trace)
|
|
962
|
+
if trace is not None:
|
|
963
|
+
tpath = os.path.join(SESSION.dir, "candidates.json")
|
|
964
|
+
counts = D.write_trace(tpath, SESSION.state["photo"], photo.shape,
|
|
965
|
+
click, res, trace)
|
|
966
|
+
trace_info = {"path": tpath, "candidates": len(trace), **counts}
|
|
729
967
|
if res is None:
|
|
730
968
|
return self._json({"found": False, "clicked": click is not None,
|
|
969
|
+
**({"trace": trace_info} if trace is not None else {}),
|
|
731
970
|
"message": ("Nothing screen-shaped was found around that "
|
|
732
971
|
"point — try clicking nearer the middle of the "
|
|
733
972
|
"screen." if click is not None else
|
|
@@ -743,6 +982,7 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
743
982
|
return self._json({"found": False,
|
|
744
983
|
"message": res["abstain_reason"],
|
|
745
984
|
"abstained": True,
|
|
985
|
+
**({"trace": trace_info} if trace is not None else {}),
|
|
746
986
|
"inspect": res["corners"]})
|
|
747
987
|
res.pop("_corners_np", None)
|
|
748
988
|
res["found"] = True
|
|
@@ -752,6 +992,8 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
752
992
|
res["confidence"] = ("corroborated" if res.get("agreement", {}).get("agree")
|
|
753
993
|
else "unconfirmed")
|
|
754
994
|
res["type_guess"] = _guess_type(res["corners"])
|
|
995
|
+
if trace is not None:
|
|
996
|
+
res["trace"] = trace_info
|
|
755
997
|
return self._json(res)
|
|
756
998
|
|
|
757
999
|
if u.path == "/api/frame":
|
|
@@ -771,6 +1013,88 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
771
1013
|
SESSION.update(fit_frame=idx)
|
|
772
1014
|
return self._json({"index": idx})
|
|
773
1015
|
|
|
1016
|
+
if u.path == "/api/preview_video":
|
|
1017
|
+
# Watch the composite move before committing to a render.
|
|
1018
|
+
#
|
|
1019
|
+
# The one-frame preview cannot answer the two questions a CLIP
|
|
1020
|
+
# raises, and they are the two settings most likely to misbehave
|
|
1021
|
+
# over time: the light match is bound ONCE from the fitted frame
|
|
1022
|
+
# (deliberately -- measuring per frame makes the screen pulse as
|
|
1023
|
+
# the UI scrolls), so a badly chosen frame is wrong for the whole
|
|
1024
|
+
# clip; and the emissive blend mixes the screenshot with the
|
|
1025
|
+
# glass beneath it, so its effect changes as the content's own
|
|
1026
|
+
# brightness does. Neither shows in a still.
|
|
1027
|
+
#
|
|
1028
|
+
# So this is a real composite through the same pipeline, at a
|
|
1029
|
+
# smaller size -- not a CSS transform over a <video>, which would
|
|
1030
|
+
# show the geometry moving and NONE of the grade, grain or blend,
|
|
1031
|
+
# which is to say none of what it is for.
|
|
1032
|
+
_need_sources()
|
|
1033
|
+
photo, ppath = _read_image(SESSION.state["photo"])
|
|
1034
|
+
spath = _safe_local_path(SESSION.state["screenshot"])
|
|
1035
|
+
if not _is_video(spath):
|
|
1036
|
+
return self._json({"error": "the screen source is not a video"}, 400)
|
|
1037
|
+
if not _have_ffmpeg():
|
|
1038
|
+
return self._json({"error": "ffmpeg is not installed",
|
|
1039
|
+
"needs_ffmpeg": True}, 400)
|
|
1040
|
+
with RENDER_LOCK:
|
|
1041
|
+
if RENDER["state"] == "running":
|
|
1042
|
+
return self._json({"error": "a render is already running"}, 409)
|
|
1043
|
+
corners = _quad(b["corners"])
|
|
1044
|
+
frac = float(b.get("radius_frac") or 0.0)
|
|
1045
|
+
fit_frame = int(b.get("fit_frame") if b.get("fit_frame") is not None
|
|
1046
|
+
else _fit_frame())
|
|
1047
|
+
first = W.read_frame_at(spath, fit_frame)
|
|
1048
|
+
radius_px = frac * first.shape[1]
|
|
1049
|
+
gr = float(b.get("grade") if b.get("grade") is not None else 0.0)
|
|
1050
|
+
grain = bool(b.get("grain", gr > 0))
|
|
1051
|
+
blend, reflection = _blend_args(b)
|
|
1052
|
+
# Downscale the PHOTO and scale the quad with it, rather than
|
|
1053
|
+
# teaching compose_video about proxies: the same code path then
|
|
1054
|
+
# produces the preview, so what is watched is what will render.
|
|
1055
|
+
# The corner radius is in SCREENSHOT pixels and does not move.
|
|
1056
|
+
scale = min(1.0, PREVIEW_WIDTH / float(photo.shape[1]))
|
|
1057
|
+
if scale < 1.0:
|
|
1058
|
+
# EVEN dimensions, both of them. H.264 with yuv420p refuses an
|
|
1059
|
+
# odd width or height, and ffmpeg's way of refusing is to die
|
|
1060
|
+
# mid-stream -- which reaches Python as a BrokenPipeError on
|
|
1061
|
+
# the frame pipe, with the real complaint nowhere in sight.
|
|
1062
|
+
# Found by measuring rather than reading: 720 x 1536/2752
|
|
1063
|
+
# rounds to 401, and the whole preview vanished.
|
|
1064
|
+
def _even(v):
|
|
1065
|
+
return max(2, int(round(v / 2.0)) * 2)
|
|
1066
|
+
nw, nh = _even(photo.shape[1] * scale), _even(photo.shape[0] * scale)
|
|
1067
|
+
# Scale the quad by what the resize ACTUALLY did, not by the
|
|
1068
|
+
# ratio that was asked for: the evening moves it by up to a
|
|
1069
|
+
# pixel, and a quad scaled by the wrong factor is a fit that
|
|
1070
|
+
# does not match the preview it is shown in.
|
|
1071
|
+
sx, sy = nw / float(photo.shape[1]), nh / float(photo.shape[0])
|
|
1072
|
+
photo = cv2.resize(photo, (nw, nh), interpolation=cv2.INTER_AREA)
|
|
1073
|
+
corners = [[x * sx, y * sy] for x, y in corners]
|
|
1074
|
+
_n, _fps, _vw, _vh = W.probe_video(spath)
|
|
1075
|
+
max_frames = max(1, int(round(PREVIEW_SECONDS * (_fps or 30))))
|
|
1076
|
+
dest = os.path.join(SESSION.dir, "preview.mp4")
|
|
1077
|
+
with RENDER_LOCK:
|
|
1078
|
+
if RENDER["state"] == "running":
|
|
1079
|
+
return self._json({"error": "a render is already running"}, 409)
|
|
1080
|
+
RENDER.update(state="running", done=0, total=0, kind="preview",
|
|
1081
|
+
output=None, message=None)
|
|
1082
|
+
try:
|
|
1083
|
+
threading.Thread(target=_render_worker, daemon=True,
|
|
1084
|
+
args=(photo, spath, corners, dest, radius_px,
|
|
1085
|
+
gr, grain, "web", fit_frame,
|
|
1086
|
+
blend, reflection, None, "preview",
|
|
1087
|
+
fit_frame, max_frames),
|
|
1088
|
+
kwargs={"smoothing": _smoothing(b)}).start()
|
|
1089
|
+
except BaseException:
|
|
1090
|
+
with RENDER_LOCK:
|
|
1091
|
+
RENDER.update(state="error", message="could not start the preview")
|
|
1092
|
+
raise
|
|
1093
|
+
return self._json({"started": True, "preview": dest,
|
|
1094
|
+
"scale": round(scale, 4),
|
|
1095
|
+
"seconds": PREVIEW_SECONDS,
|
|
1096
|
+
"from_frame": fit_frame})
|
|
1097
|
+
|
|
774
1098
|
if u.path == "/api/render":
|
|
775
1099
|
# Video: same fit, same geometry, N frames instead of one.
|
|
776
1100
|
_need_sources()
|
|
@@ -812,9 +1136,18 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
812
1136
|
# undercuts the determinism claim.
|
|
813
1137
|
result = {"output": dest, "photo": ppath, "screenshot": spath,
|
|
814
1138
|
"corners": corners, "radius_frac": frac, "radius_px": radius_px,
|
|
815
|
-
"device": b.get("device"), "
|
|
1139
|
+
"device": b.get("device"), "corner_smoothing": _smoothing(b),
|
|
1140
|
+
"grade": gr, "grain": grain,
|
|
816
1141
|
"video": True, "preset": preset, "fit_frame": fit_frame,
|
|
817
|
-
"blend": blend, "reflection": reflection
|
|
1142
|
+
"blend": blend, "reflection": reflection,
|
|
1143
|
+
# A render is always the whole clip; only the preview
|
|
1144
|
+
# passes a segment. Recorded anyway, because the
|
|
1145
|
+
# sidecar's promise is EVERY argument that changes the
|
|
1146
|
+
# output — and if rendering a segment ever ships, the
|
|
1147
|
+
# recipe already carries it rather than silently
|
|
1148
|
+
# reproducing something else. test_sidecar.py caught
|
|
1149
|
+
# their absence the moment they were added.
|
|
1150
|
+
"start_frame": 0, "max_frames": None}
|
|
818
1151
|
# `state="running"` means "a thread is running", so it is set
|
|
819
1152
|
# here -- after every line that can raise, immediately before the
|
|
820
1153
|
# thread exists. It used to be set at the top of this route, so a
|
|
@@ -826,13 +1159,14 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
826
1159
|
with RENDER_LOCK:
|
|
827
1160
|
if RENDER["state"] == "running":
|
|
828
1161
|
return self._json({"error": "a render is already running"}, 409)
|
|
829
|
-
RENDER.update(state="running", done=0, total=0,
|
|
1162
|
+
RENDER.update(state="running", done=0, total=0, kind="render",
|
|
830
1163
|
output=None, message=None)
|
|
831
1164
|
try:
|
|
832
1165
|
threading.Thread(target=_render_worker, daemon=True,
|
|
833
1166
|
args=(photo, spath, corners, dest, radius_px,
|
|
834
1167
|
gr, grain, preset, fit_frame,
|
|
835
|
-
blend, reflection, result)
|
|
1168
|
+
blend, reflection, result),
|
|
1169
|
+
kwargs={"smoothing": _smoothing(b)}).start()
|
|
836
1170
|
except BaseException:
|
|
837
1171
|
# If the thread cannot even be created, the flag must not
|
|
838
1172
|
# outlive the request.
|
|
@@ -854,7 +1188,9 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
854
1188
|
# when the photograph is (a portfolio shot).
|
|
855
1189
|
gr = float(b.get("grade") if b.get("grade") is not None else 0.0)
|
|
856
1190
|
blend, reflection = _blend_args(b)
|
|
1191
|
+
smoothing = _smoothing(b)
|
|
857
1192
|
out = W.compose(photo, shot, corners, radius_px,
|
|
1193
|
+
corner_smoothing=smoothing,
|
|
858
1194
|
grade=gr, grain=bool(b.get("grain", gr > 0)),
|
|
859
1195
|
blend=blend, reflection=reflection)
|
|
860
1196
|
SESSION.update(corners=corners, radius_frac=frac, device=b.get("device"),
|
|
@@ -889,9 +1225,29 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
889
1225
|
# cannot be forgotten the same way.
|
|
890
1226
|
result = {"output": dest, "photo": ppath, "screenshot": spath, "corners": corners,
|
|
891
1227
|
"radius_frac": frac, "radius_px": radius_px, "device": b.get("device"),
|
|
1228
|
+
"corner_smoothing": smoothing,
|
|
892
1229
|
"grade": gr, "grain": bool(b.get("grain", gr > 0)),
|
|
893
1230
|
"blend": blend, "reflection": reflection,
|
|
894
1231
|
"saved": time.time()}
|
|
1232
|
+
# A fit is remembered when it PRODUCED something, not while it
|
|
1233
|
+
# is being dragged: a quad on the canvas is a work in progress,
|
|
1234
|
+
# a quad that made an output is one the person looked at and
|
|
1235
|
+
# kept. Next run on this photograph starts from it.
|
|
1236
|
+
key = FIT.key_for(photo)
|
|
1237
|
+
FIT.remember(key, corners, frac, b.get("device"),
|
|
1238
|
+
ppath, (photo.shape[1], photo.shape[0]))
|
|
1239
|
+
# ...and the portable half: a file beside the mockup, in the
|
|
1240
|
+
# folder the user already chose. The store above is invisible and
|
|
1241
|
+
# keyed on this machine's copy of the pixels; this one can be
|
|
1242
|
+
# found in Finder, kept with the project, sent to someone, and
|
|
1243
|
+
# dragged back in against a re-export the key would miss.
|
|
1244
|
+
result["fit_file"] = FF.write(dest, FF.build(
|
|
1245
|
+
corners, frac, b.get("device"), ppath,
|
|
1246
|
+
(photo.shape[1], photo.shape[0]), key))
|
|
1247
|
+
# Sidecar first, THEN the session output: /api/import reads that
|
|
1248
|
+
# pointer and may be called the moment this returns, and the
|
|
1249
|
+
# bug template asks for the sidecar first. Same publication order
|
|
1250
|
+
# the render worker uses, and for the same reason.
|
|
895
1251
|
_write_json_atomic(SESSION.result_path, result)
|
|
896
1252
|
SESSION.update(output=dest)
|
|
897
1253
|
return self._json(result)
|