retake-demos 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -0
- package/LICENSE +21 -0
- package/README.md +183 -0
- package/assets/icon-192.png +0 -0
- package/assets/icon-512.png +0 -0
- package/assets/logo.svg +13 -0
- package/demos/example.yaml +65 -0
- package/dist/appserver.js +316 -0
- package/dist/cli.js +467 -0
- package/dist/describe.js +377 -0
- package/dist/digest.js +197 -0
- package/dist/dryrun.js +196 -0
- package/dist/edits.js +229 -0
- package/dist/manifest.js +358 -0
- package/dist/operator/run.js +231 -0
- package/dist/operator/tools.js +412 -0
- package/dist/paths.js +39 -0
- package/dist/presets.js +75 -0
- package/dist/record.js +814 -0
- package/dist/render.js +661 -0
- package/dist/ui/chat.html +385 -0
- package/dist/ui/guide.html +85 -0
- package/dist/ui/index.html +722 -0
- package/dist/ui/server.js +1112 -0
- package/package.json +67 -0
- package/site/index.html +158 -0
- package/skill/SKILL.md +154 -0
package/dist/manifest.js
ADDED
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The demo manifest — the one file a human (or later, an agent) writes.
|
|
3
|
+
*
|
|
4
|
+
* It is a superset of a testreel recording definition: the browser/steps
|
|
5
|
+
* vocabulary is testreel's, plus Retake's own layers on top:
|
|
6
|
+
* - `seed` what state the app should be in before the camera rolls
|
|
7
|
+
* - `scene` step markers that become burned-in captions + the proof log
|
|
8
|
+
* - `outputs` which artifacts to produce and how
|
|
9
|
+
*/
|
|
10
|
+
import fs from "node:fs";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import YAML from "yaml";
|
|
13
|
+
import { z } from "zod";
|
|
14
|
+
import { DEFAULT_PRESET, PRESETS } from "./presets.js";
|
|
15
|
+
const Selector = z.string().min(1);
|
|
16
|
+
/** Where a pointer goes: a selector (its centre), a raw viewport point, or a
|
|
17
|
+
selector plus an offset. Points are resolved by reading the rect in the
|
|
18
|
+
page rather than through Playwright's locator engine, because canvas
|
|
19
|
+
editors (Blockly, PixiJS) intercept pointer events and make ordinary
|
|
20
|
+
locators time out. */
|
|
21
|
+
export const Point = z.union([
|
|
22
|
+
Selector,
|
|
23
|
+
z.object({ x: z.number(), y: z.number() }),
|
|
24
|
+
z.object({ selector: Selector, dx: z.number().default(0), dy: z.number().default(0) }),
|
|
25
|
+
]);
|
|
26
|
+
const Base = z.object({
|
|
27
|
+
/** ms to rest after the step. */
|
|
28
|
+
pauseAfter: z.number().nonnegative().optional(),
|
|
29
|
+
/** Selector to wait for before running the step. */
|
|
30
|
+
waitFor: Selector.optional(),
|
|
31
|
+
timeout: z.number().positive().optional(),
|
|
32
|
+
/** This step handles a secret. Its text is never logged, never written to the
|
|
33
|
+
proof log, and never echoed to the UI. Put the value in .env and reference
|
|
34
|
+
it as ${VAR} — do not paste passwords into a manifest. */
|
|
35
|
+
secret: z.boolean().optional(),
|
|
36
|
+
});
|
|
37
|
+
export const Step = z.discriminatedUnion("action", [
|
|
38
|
+
Base.extend({ action: z.literal("wait"), ms: z.number().nonnegative().default(1000) }),
|
|
39
|
+
Base.extend({ action: z.literal("click"), selector: Selector.optional(), at: Point.optional(), zoom: z.number().positive().optional() }),
|
|
40
|
+
Base.extend({
|
|
41
|
+
action: z.literal("type"),
|
|
42
|
+
selector: Selector,
|
|
43
|
+
text: z.string(),
|
|
44
|
+
delay: z.number().nonnegative().optional(),
|
|
45
|
+
clear: z.boolean().optional(),
|
|
46
|
+
}),
|
|
47
|
+
Base.extend({ action: z.literal("fill"), selector: Selector, text: z.string() }),
|
|
48
|
+
Base.extend({ action: z.literal("hover"), selector: Selector.optional(), at: Point.optional() }),
|
|
49
|
+
/** Scroll by a pixel delta, or `to` a selector — which computes the delta so
|
|
50
|
+
the element lands where you want it and animates there, so the viewer sees
|
|
51
|
+
the page move rather than jumping. */
|
|
52
|
+
/** Press at one point, move, release: the gesture that block editors,
|
|
53
|
+
canvases and sortable lists are built on. The cursor overlay gets two
|
|
54
|
+
keyframes (start, end) while the page gets many small moves, so a drag
|
|
55
|
+
costs the overlay no more than a click does. */
|
|
56
|
+
Base.extend({
|
|
57
|
+
action: z.literal("drag"),
|
|
58
|
+
from: Point,
|
|
59
|
+
to: Point,
|
|
60
|
+
/** Intermediate mouse moves. The page needs several to believe a drag. */
|
|
61
|
+
steps: z.number().int().min(4).max(60).default(16),
|
|
62
|
+
/** Pause after pressing, before moving. Some editors need it. */
|
|
63
|
+
holdMs: z.number().int().min(0).max(2000).default(120),
|
|
64
|
+
/** How long the move takes, ms. */
|
|
65
|
+
durationMs: z.number().int().min(120).max(6000).default(700),
|
|
66
|
+
}),
|
|
67
|
+
Base.extend({
|
|
68
|
+
action: z.literal("scroll"),
|
|
69
|
+
x: z.number().optional(),
|
|
70
|
+
y: z.number().optional(),
|
|
71
|
+
/** A selector to bring into view, or "top" / "bottom" of the page.
|
|
72
|
+
(x/y are a relative nudge in pixels — y: 0 moves nothing.) */
|
|
73
|
+
to: z.union([z.literal("top"), z.literal("bottom"), Selector]).optional(),
|
|
74
|
+
/** Where the target should end up: top | center (default) | bottom. */
|
|
75
|
+
align: z.enum(["top", "center", "bottom"]).default("center"),
|
|
76
|
+
/** How fast to move. 1 is the default 600ms; below 1 is slower, so
|
|
77
|
+
`speed: 0.4` takes 1.5s. A long scroll at full speed reads as a jump —
|
|
78
|
+
slow it down and the viewer can follow what went past. */
|
|
79
|
+
speed: z.number().positive().max(4).optional(),
|
|
80
|
+
}),
|
|
81
|
+
Base.extend({
|
|
82
|
+
action: z.literal("zoom"),
|
|
83
|
+
selector: Selector.optional(),
|
|
84
|
+
scale: z.number().positive().default(2),
|
|
85
|
+
x: z.number().optional(),
|
|
86
|
+
y: z.number().optional(),
|
|
87
|
+
duration: z.number().positive().optional(),
|
|
88
|
+
}),
|
|
89
|
+
Base.extend({ action: z.literal("keyboard"), key: z.string().min(1) }),
|
|
90
|
+
Base.extend({ action: z.literal("navigate"), url: z.string().min(1) }),
|
|
91
|
+
Base.extend({ action: z.literal("screenshot"), name: z.string().optional() }),
|
|
92
|
+
/** Attach local files to a file input (no OS dialog is involved). */
|
|
93
|
+
Base.extend({ action: z.literal("upload"), selector: Selector, files: z.union([z.string(), z.array(z.string())]) }),
|
|
94
|
+
/** Click something that downloads, and catch the file. Saved next to the
|
|
95
|
+
video as downloads/<name>; `expect` fails the step if the name mismatches. */
|
|
96
|
+
Base.extend({ action: z.literal("download"), selector: Selector.optional(), saveAs: z.string().optional(), expect: z.string().optional() }),
|
|
97
|
+
/** Wait for a selector to appear (Retake extra; the thing testreel's JSON lacks
|
|
98
|
+
for AI-driven apps whose next screen arrives whenever the model answers). */
|
|
99
|
+
Base.extend({ action: z.literal("waitFor"), selector: Selector }),
|
|
100
|
+
/** Run JS in the page (Retake extra). Used for seeding, hiding chrome, etc. */
|
|
101
|
+
Base.extend({ action: z.literal("evaluate"), script: z.string().min(1) }),
|
|
102
|
+
/** Change what a stubbed endpoint answers, mid-take. This is how a demo shows
|
|
103
|
+
something *arriving*: the queue is empty, an action happens, the stub is
|
|
104
|
+
swapped, the next poll shows the new row. */
|
|
105
|
+
Base.extend({
|
|
106
|
+
action: z.literal("stub"),
|
|
107
|
+
url: z.string().min(1),
|
|
108
|
+
method: z.enum(["GET", "POST", "PUT", "PATCH", "DELETE"]).optional(),
|
|
109
|
+
status: z.number().int().optional(),
|
|
110
|
+
json: z.unknown().optional(),
|
|
111
|
+
from: z.string().optional(),
|
|
112
|
+
}),
|
|
113
|
+
/** A named beat. Timestamps go into the proof log; `caption` gets burned into
|
|
114
|
+
the video from this moment until the next scene (or `holdMs`). */
|
|
115
|
+
Base.extend({
|
|
116
|
+
action: z.literal("scene"),
|
|
117
|
+
label: z.string().min(1),
|
|
118
|
+
caption: z.string().optional(),
|
|
119
|
+
holdMs: z.number().positive().optional(),
|
|
120
|
+
/** Where the camera looks during this scene. Omitted → the manifest's
|
|
121
|
+
`camera` policy (auto = focus on the last selector used, static = none). */
|
|
122
|
+
camera: z
|
|
123
|
+
.union([z.literal("static"), z.literal("auto"), z.object({ focus: Selector.optional(), zoom: z.number().min(1).max(3).optional() })])
|
|
124
|
+
.optional(),
|
|
125
|
+
}),
|
|
126
|
+
]).superRefine((st, ctx) => {
|
|
127
|
+
if ((st.action === "click" || st.action === "hover") && !st.selector && !st.at) {
|
|
128
|
+
ctx.addIssue({ code: "custom", message: `${st.action} needs either a selector or an \`at\` point` });
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
/** Answer a network call with canned data for the length of a take.
|
|
132
|
+
Lets a demo show populated screens when the real backend is unavailable,
|
|
133
|
+
rate-limited, or full of data you must not put on camera. Every stubbed
|
|
134
|
+
pattern is named in the proof log — a take never hides that it was faked. */
|
|
135
|
+
export const Stub = z.object({
|
|
136
|
+
/** Glob or regex-ish pattern, Playwright style: "**\/api/requests*". */
|
|
137
|
+
url: z.string().min(1),
|
|
138
|
+
/** Only intercept this method; other methods pass through to the app. */
|
|
139
|
+
method: z.enum(["GET", "POST", "PUT", "PATCH", "DELETE"]).optional(),
|
|
140
|
+
status: z.number().int().default(200),
|
|
141
|
+
/** Inline JSON, or a file next to the manifest. One of the two. */
|
|
142
|
+
json: z.unknown().optional(),
|
|
143
|
+
from: z.string().optional(),
|
|
144
|
+
contentType: z.string().default("application/json; charset=utf-8"),
|
|
145
|
+
});
|
|
146
|
+
export const Seed = z.discriminatedUnion("kind", [
|
|
147
|
+
/** Write a JSON document to a file before the app is touched. Good for apps
|
|
148
|
+
with a file-backed sync hub (Capture: SYNC_DATA_DIR/sync.json). */
|
|
149
|
+
z.object({
|
|
150
|
+
kind: z.literal("file"), path: z.string(), from: z.string(),
|
|
151
|
+
/** Resolve "-12d", "+6h", "now" strings in the JSON to epoch ms at seed
|
|
152
|
+
time, so a fixture that says "three days ago" still says it next
|
|
153
|
+
month. Every app with relative dates in its UI needs this. */
|
|
154
|
+
relativeDates: z.boolean().default(false),
|
|
155
|
+
}),
|
|
156
|
+
/** Run JS in the page before recording starts, then reload. Good for
|
|
157
|
+
IndexedDB / localStorage-backed apps. `from` is a JSON file exposed as
|
|
158
|
+
`window.__seed` for the script. */
|
|
159
|
+
z.object({ kind: z.literal("evaluate"), script: z.string(), from: z.string().optional() }),
|
|
160
|
+
/** Shell command run before recording (e.g. `curl -X POST …/reset`). */
|
|
161
|
+
z.object({ kind: z.literal("command"), run: z.string() }),
|
|
162
|
+
]);
|
|
163
|
+
const ManifestShape = z.object({
|
|
164
|
+
name: z.string().regex(/^[a-z0-9-]+$/, "kebab-case only"),
|
|
165
|
+
title: z.string().optional(),
|
|
166
|
+
url: z.string().url(),
|
|
167
|
+
/** Quality preset — the publishing format. Fields below override it. */
|
|
168
|
+
preset: z.enum(Object.keys(PRESETS)).default(DEFAULT_PRESET),
|
|
169
|
+
/** Recording viewport (video pixels). Default = the preset's canvas. Give a
|
|
170
|
+
narrow app a narrower viewport (e.g. 1240×1080 for a 620px column at 2×)
|
|
171
|
+
and the card layout centres it in the canvas. */
|
|
172
|
+
viewport: z.object({ width: z.number().int(), height: z.number().int() }).optional(),
|
|
173
|
+
/** CSS zoom applied to the page (2 → app sees half the canvas in CSS px, renders crisp). */
|
|
174
|
+
scale: z.number().min(0.5).max(4).optional(),
|
|
175
|
+
fps: z.number().int().min(10).max(60).optional(),
|
|
176
|
+
crf: z.number().int().min(10).max(35).optional(),
|
|
177
|
+
/** band (default): the site fills the frame, caption strip below · card: the recording framed on a soft canvas · overlay-*: caption over the video · none */
|
|
178
|
+
layout: z.enum(["band", "card", "overlay-bottom", "overlay-top", "none"]).optional(),
|
|
179
|
+
/** auto: each scene zooms toward the last element the demo touched · static: no camera moves. */
|
|
180
|
+
camera: z.union([z.literal("auto"), z.literal("static"), z.object({ zoom: z.number().min(1).max(3) })]).default("auto"),
|
|
181
|
+
/** Seconds cut from the head/tail of the finished video. Render-only: change
|
|
182
|
+
it and re-render, no re-recording. */
|
|
183
|
+
trim: z.object({ head: z.number().nonnegative().default(0), tail: z.number().nonnegative().default(0) }).prefault({}),
|
|
184
|
+
/** Sign in once, reuse the session. `storageState` is a Playwright state file
|
|
185
|
+
(cookies + localStorage) written after `setup` and loaded on later runs, so
|
|
186
|
+
the login only happens when the file is missing or stale. */
|
|
187
|
+
auth: z
|
|
188
|
+
.object({
|
|
189
|
+
storageState: z.string(),
|
|
190
|
+
maxAgeHours: z.number().positive().default(72),
|
|
191
|
+
/** The sign-in steps themselves. These are the ONLY steps skipped when a
|
|
192
|
+
saved session is reused — `setup` always runs, because it usually
|
|
193
|
+
contains things that have nothing to do with logging in. */
|
|
194
|
+
setup: z.array(Step).default([]),
|
|
195
|
+
})
|
|
196
|
+
.optional(),
|
|
197
|
+
captions: z.union([z.boolean(), z.object({ fontSize: z.number().int().optional(), color: z.string().optional() })]).default(true),
|
|
198
|
+
theme: z.object({ background: z.string().optional(), ink: z.string().optional() }).prefault({}),
|
|
199
|
+
colorScheme: z.enum(["light", "dark"]).default("light"),
|
|
200
|
+
/** Ask the page for reduced motion. Apps that respect it lose their infinite
|
|
201
|
+
spinners and entrance animations — steadier frames, smaller files. */
|
|
202
|
+
reducedMotion: z.boolean().default(false),
|
|
203
|
+
waitForSelector: Selector.optional(),
|
|
204
|
+
speed: z.number().positive().default(1),
|
|
205
|
+
/** Cursor overlay (testreel). false → none; {style: touch} → tap dot. Size is preset-scaled unless set. */
|
|
206
|
+
/** cursor.idleHide: fade the cursor out when idle (testreel default). Each
|
|
207
|
+
idle gap adds a nested if() to the ffmpeg alpha expression, and ffmpeg's
|
|
208
|
+
parser has a depth limit — long demos blow past it. Retake turns it off
|
|
209
|
+
automatically above 60 steps; set it explicitly either way. */
|
|
210
|
+
cursor: z.union([z.boolean(), z.object({ style: z.enum(["default", "pointer", "text", "touch"]).optional(), size: z.number().optional(), idleHideMs: z.number().optional(), idleHide: z.boolean().optional() })]).default(true),
|
|
211
|
+
seed: z.array(Seed).default([]),
|
|
212
|
+
/** What a failed step does to the take. "stop" (default): the camera stops
|
|
213
|
+
at the failure, so nobody gets one interaction and ten minutes of
|
|
214
|
+
nothing. "continue": keep rolling — for demos that tolerate a miss. */
|
|
215
|
+
/** A demo that opens a second tab loses its subject: Retake records one
|
|
216
|
+
page. By default new tabs are folded back into this one (window.open and
|
|
217
|
+
target=_blank navigate in place), so the flow stays on camera. Set false
|
|
218
|
+
when a demo genuinely needs a separate window. */
|
|
219
|
+
keepInTab: z.boolean().default(true),
|
|
220
|
+
/** Demos that share live state (one Capture hub, one test account) must
|
|
221
|
+
not run at once: the second take's seed wipes the first take's board
|
|
222
|
+
mid-video. Name the shared thing; takes holding the same name queue. */
|
|
223
|
+
lock: z.string().regex(/^[a-z0-9-]+$/).optional(),
|
|
224
|
+
onFail: z.enum(["stop", "continue"]).default("stop"),
|
|
225
|
+
/** Playback tempo applied at render time: 0.8 slows the whole take for a
|
|
226
|
+
client who needs it gentler; 1.2 tightens it. Cursor and clicks stay in
|
|
227
|
+
sync by construction (they are pixels in the recording); captions are
|
|
228
|
+
retimed to match. Re-render only — never a re-record. */
|
|
229
|
+
tempo: z.number().min(0.5).max(2).default(1),
|
|
230
|
+
/** Hard ceiling on a take, seconds. Unset, it scales with the demo
|
|
231
|
+
(≈10s per step plus the explicit waits, never under 240s): the cap is
|
|
232
|
+
for a take that is stuck, not one that is long. Set it to override. */
|
|
233
|
+
maxSeconds: z.number().int().min(10).max(3600).optional(),
|
|
234
|
+
/** Canned network responses, armed before the first navigation. */
|
|
235
|
+
stub: z.array(Stub).default([]),
|
|
236
|
+
/** Steps run before the camera rolls. */
|
|
237
|
+
setup: z.array(Step).default([]),
|
|
238
|
+
steps: z.array(Step).min(1),
|
|
239
|
+
outputs: z
|
|
240
|
+
.object({
|
|
241
|
+
mp4: z.boolean().default(true),
|
|
242
|
+
/** GIF is opt-in (secondary): true → preset's GIF settings (or 720px/15fps), or give width/fps. */
|
|
243
|
+
gif: z.union([z.boolean(), z.object({ width: z.number().int().optional(), fps: z.number().int().optional() })]).default(false),
|
|
244
|
+
/** Which scene's frame becomes the thumbnail; default = last scene. */
|
|
245
|
+
thumbnail: z.union([z.boolean(), z.object({ scene: z.string() })]).default(true),
|
|
246
|
+
/** One PNG per scene, into outputs/<name>/stills/. */
|
|
247
|
+
stills: z.boolean().default(true),
|
|
248
|
+
})
|
|
249
|
+
.prefault({}),
|
|
250
|
+
});
|
|
251
|
+
export const Manifest = ManifestShape.superRefine((m, ctx) => {
|
|
252
|
+
[...m.steps, ...m.setup].forEach((st, i) => {
|
|
253
|
+
if ((st.action === "click" || st.action === "hover") && !st.selector && !st.at) {
|
|
254
|
+
ctx.addIssue({ code: "custom", path: ["steps", i], message: `${st.action} needs either a selector or an \`at\` point` });
|
|
255
|
+
}
|
|
256
|
+
});
|
|
257
|
+
// Scenes are addressed by label (thumbnail, captions, camera, stills,
|
|
258
|
+
// until); two with the same label make every one of those ambiguous.
|
|
259
|
+
const seen = new Map();
|
|
260
|
+
m.steps.forEach((st, i) => {
|
|
261
|
+
if (st.action !== "scene")
|
|
262
|
+
return;
|
|
263
|
+
if (seen.has(st.label))
|
|
264
|
+
ctx.addIssue({ code: "custom", path: ["steps", i, "label"], message: `duplicate scene label "${st.label}" (also step ${seen.get(st.label)}) — labels must be unique` });
|
|
265
|
+
else
|
|
266
|
+
seen.set(st.label, i);
|
|
267
|
+
});
|
|
268
|
+
});
|
|
269
|
+
export function loadManifest(file) {
|
|
270
|
+
const abs = path.resolve(file);
|
|
271
|
+
const raw = fs.readFileSync(abs, "utf8");
|
|
272
|
+
let data;
|
|
273
|
+
try {
|
|
274
|
+
data = abs.endsWith(".json") ? JSON.parse(raw) : YAML.parse(raw);
|
|
275
|
+
}
|
|
276
|
+
catch (e) {
|
|
277
|
+
// The classic trap: a one-line JS object literal under `script:` is a
|
|
278
|
+
// YAML mapping, and the parser fails before validation ever runs. Find
|
|
279
|
+
// the line it choked on; if that line carries a script, say so.
|
|
280
|
+
const err = e;
|
|
281
|
+
const line = err.linePos?.[0]?.line;
|
|
282
|
+
const src = line ? raw.split("\n")[line - 1] ?? "" : "";
|
|
283
|
+
const hint = /\bscript:/.test(src) || /compact mappings/.test(err.message) ? "\n hint: a one-line script with braces parses as YAML, not JS — write it as a block:\n script: |\n window.scrollTo({ top: 0 })" : "";
|
|
284
|
+
throw new Error(`Could not parse ${file}: ${err.message.split("\n")[0]}${hint}`);
|
|
285
|
+
}
|
|
286
|
+
const parsed = Manifest.safeParse(data);
|
|
287
|
+
if (!parsed.success) {
|
|
288
|
+
const issues = parsed.error.issues.map((i) => {
|
|
289
|
+
const where = i.path.join(".") || "(root)";
|
|
290
|
+
// The classic trap: a one-line JS object literal under `script:` parses
|
|
291
|
+
// as a YAML mapping. Name it, or everyone hits it once.
|
|
292
|
+
const hint = where.endsWith(".script") && /expected string/i.test(i.message) ? " — a one-line script with braces parses as YAML; write it as a block: script: |" : "";
|
|
293
|
+
return ` - ${where}: ${i.message}${hint}`;
|
|
294
|
+
}).join("\n");
|
|
295
|
+
throw new Error(`Invalid manifest ${file}:\n${issues}`);
|
|
296
|
+
}
|
|
297
|
+
return { manifest: parsed.data, file: abs, dir: path.dirname(abs) };
|
|
298
|
+
}
|
|
299
|
+
/** Things worth saying out loud before a run. Not errors — the run proceeds. */
|
|
300
|
+
export function warnings(m) {
|
|
301
|
+
const w = [];
|
|
302
|
+
// Secrets typed during recorded steps end up in the video. In `setup` they do
|
|
303
|
+
// not: setup runs before the camera and is trimmed off the front.
|
|
304
|
+
const onCamera = m.steps.filter((s) => s.secret).length;
|
|
305
|
+
if (onCamera)
|
|
306
|
+
w.push(`${onCamera} step(s) marked secret are in \`steps\`, not \`setup\` — redaction hides them from the logs, but they will be VISIBLE IN THE VIDEO. Move the sign-in into \`setup\`.`);
|
|
307
|
+
for (const s of [...m.steps, ...m.setup]) {
|
|
308
|
+
if ((s.action === "type" || s.action === "fill") && !s.secret && /pass|secret|token|api[-_ ]?key/i.test(s.selector)) {
|
|
309
|
+
w.push(`step "${s.action} ${s.selector}" looks like a credential field but is not marked \`secret: true\` — its text will be logged.`);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
if (m.auth && !m.auth.setup.length && !m.setup.length)
|
|
313
|
+
w.push("`auth.storageState` is set but there are no sign-in steps under `auth.setup` — there is nothing to save a session from.");
|
|
314
|
+
const moves = m.steps.filter((st) => ["click", "type", "fill", "hover", "scroll", "upload"].includes(st.action)).length;
|
|
315
|
+
if (m.cursor !== false && moves > 45)
|
|
316
|
+
w.push(`~${moves} cursor moves — the cursor overlay (testreel/ffmpeg) cannot exceed ~45 in one take and will be MISSING from the video. Split the demo into shorter ones, or set \`cursor: false\`.`);
|
|
317
|
+
m.steps.forEach((st, i) => {
|
|
318
|
+
if (st.action !== "scene")
|
|
319
|
+
return;
|
|
320
|
+
const rest = m.steps.slice(i + 1);
|
|
321
|
+
const nextScene = rest.findIndex((x) => x.action === "scene");
|
|
322
|
+
const between = nextScene === -1 ? rest : rest.slice(0, nextScene);
|
|
323
|
+
// A closing hold on the payoff is normal; only a waits-only scene *followed by another* is empty.
|
|
324
|
+
if (nextScene !== -1 && between.length && between.every((x) => x.action === "wait"))
|
|
325
|
+
w.push(`scene "${st.label}" has only waits before the next scene — nothing happens on camera in it.`);
|
|
326
|
+
});
|
|
327
|
+
return w;
|
|
328
|
+
}
|
|
329
|
+
/** `${VAR}` substitution against process.env, like testreel. */
|
|
330
|
+
export function expandEnv(s) {
|
|
331
|
+
return s.replace(/\$\{([A-Z0-9_]+)\}/g, (_, k) => {
|
|
332
|
+
const v = process.env[k];
|
|
333
|
+
if (v === undefined)
|
|
334
|
+
throw new Error(`manifest references \${${k}} but it is not set`);
|
|
335
|
+
return v;
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
export function resolve(m) {
|
|
339
|
+
const p = PRESETS[m.preset] ?? PRESETS[DEFAULT_PRESET];
|
|
340
|
+
const viewport = { width: m.viewport?.width ?? p.width, height: m.viewport?.height ?? p.height };
|
|
341
|
+
const gifOverride = m.outputs.gif;
|
|
342
|
+
const gif = gifOverride === false ? false : gifOverride === true ? p.gif : p.gif === false ? { width: gifOverride.width ?? 720, fps: gifOverride.fps ?? 15 } : { width: gifOverride.width ?? p.gif.width, fps: gifOverride.fps ?? p.gif.fps };
|
|
343
|
+
const cursor = m.cursor === false ? false : { style: (m.cursor === true ? "default" : m.cursor.style ?? "default"), size: m.cursor === true ? p.cursorSize : m.cursor.size ?? p.cursorSize, idleHideMs: m.cursor === true ? 2000 : m.cursor.idleHideMs ?? 2000, idleHide: m.cursor === true ? undefined : m.cursor.idleHide };
|
|
344
|
+
const captions = m.captions === false ? false : { fontSize: m.captions === true ? p.captionFontSize : m.captions.fontSize ?? p.captionFontSize, color: m.captions === true ? undefined : m.captions.color };
|
|
345
|
+
return {
|
|
346
|
+
...p,
|
|
347
|
+
viewport,
|
|
348
|
+
scale: m.scale ?? p.scale,
|
|
349
|
+
fps: m.fps ?? p.fps,
|
|
350
|
+
crf: m.crf ?? p.crf,
|
|
351
|
+
layout: m.layout ?? p.layout,
|
|
352
|
+
cameraZoom: typeof m.camera === "object" ? m.camera.zoom : p.cameraZoom,
|
|
353
|
+
gif,
|
|
354
|
+
cursor,
|
|
355
|
+
captions,
|
|
356
|
+
theme: { background: m.theme.background ?? "#e6e9e2", ink: m.theme.ink ?? "#191d19" },
|
|
357
|
+
};
|
|
358
|
+
}
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Operator sessions: the signed-in coding agent driving Retake, fenced to
|
|
3
|
+
* Retake's MCP tools, narrating to the UI, asking only when stuck.
|
|
4
|
+
*
|
|
5
|
+
* One session = one "make me a demo" request. The agent's shell is denied;
|
|
6
|
+
* its only hands are src/operator/tools.ts. Approvals (start the app) and
|
|
7
|
+
* questions come back here as pending items the UI shows, and the tool call
|
|
8
|
+
* blocks until the person answers.
|
|
9
|
+
*/
|
|
10
|
+
import fs from "node:fs";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import { execFileSync, spawn } from "node:child_process";
|
|
13
|
+
import { entry } from "../paths.js";
|
|
14
|
+
const sessions = new Map();
|
|
15
|
+
export const getSession = (id) => sessions.get(id);
|
|
16
|
+
function push(s, type, data) {
|
|
17
|
+
// Keep everything a person might want to re-read; the UI decides what to show.
|
|
18
|
+
if (type === "line")
|
|
19
|
+
s.lines.push(String(data));
|
|
20
|
+
else if (type === "tool")
|
|
21
|
+
s.lines.push(`→ ${String(data)}`);
|
|
22
|
+
else if (type === "thought")
|
|
23
|
+
s.lines.push(` ${String(data)}`);
|
|
24
|
+
for (const l of s.listeners)
|
|
25
|
+
l({ type, data });
|
|
26
|
+
}
|
|
27
|
+
/** What the operator is told it is, and how to behave. Short on purpose: the
|
|
28
|
+
tools carry their own instructions, and rules beat prose. */
|
|
29
|
+
export function brief(input) {
|
|
30
|
+
return [
|
|
31
|
+
"You are Retake's operator. A person wants a demo video of their web app. You have Retake's tools and nothing else — no shell.",
|
|
32
|
+
"",
|
|
33
|
+
`What they want: ${input.describe}`,
|
|
34
|
+
input.url ? `Their app is at: ${input.url}` : "They did not give a URL.",
|
|
35
|
+
input.project ? `Their app's source is in: ${input.project}` : "No source folder was given.",
|
|
36
|
+
input.name ? `Use the demo name: ${input.name}` : "Pick a short kebab-case demo name from what they asked for.",
|
|
37
|
+
"",
|
|
38
|
+
"How to work — the order that wins:",
|
|
39
|
+
"1. If you have a folder, read_project first. It tells you routes, the start command, sign-in fields, selectors, and what animates forever.",
|
|
40
|
+
"2. Make sure the app answers: scout the URL. If it refuses, check ports — dev servers often come up one port over. If it is really down and you have the folder and its start command, start_app immediately; a request made inside Retake already authorizes starting that selected project. If you cannot tell which app or port is meant, ask — one question, with what you found.",
|
|
41
|
+
"3. draft the manifest from the sentence (or write_manifest yourself if you know better). Read the returned YAML. Make sure its recorded steps actually perform every requested interaction — scene and wait steps only narrate/pause and never substitute for clicks, typing, or navigation. Fix it with write_manifest or edit: wait for results not timers, unique selectors, logins under auth.setup with ${ENV} placeholders and secret: true, reducedMotion if things animate forever, scroll to long forms.",
|
|
42
|
+
"4. dry. Always before run. Fix what it reports and dry again. Do not run a manifest whose dry run fails.",
|
|
43
|
+
"5. run with preview=true. Read the receipts. If a step failed or the story is wrong, edit and run again. Two or three rounds is normal; more than four means ask the person.",
|
|
44
|
+
"6. When it is good, run with preview=false once for the real quality, then done with one sentence.",
|
|
45
|
+
"",
|
|
46
|
+
"Rules:",
|
|
47
|
+
"- Never type real passwords into a manifest; use ${ENV_VAR} and secret: true, and tell the person which variable to set.",
|
|
48
|
+
"- Never invent selectors; use what scout/read_project returned.",
|
|
49
|
+
"- Never accept a static manifest for an interactive request. It must perform the requested actions after setup, not merely describe them with scenes and waits.",
|
|
50
|
+
"- Narrate nothing yourself — the tools already tell the person what is happening. Your text is for thinking; keep it short.",
|
|
51
|
+
"- If the backend is unavailable, prefer a stub: block in the manifest over giving up; say so in done.",
|
|
52
|
+
].join("\n");
|
|
53
|
+
}
|
|
54
|
+
export function startOperator(input) {
|
|
55
|
+
const id = "op" + Date.now().toString(36);
|
|
56
|
+
const s = { id, proc: null, lines: [], pending: [], done: false, code: null, onDemo: input.onDemo, startedAt: Date.now(), listeners: new Set() };
|
|
57
|
+
sessions.set(id, s);
|
|
58
|
+
const tools = entry("operator/tools");
|
|
59
|
+
const toolEnv = { RETAKE_ROOT: input.root, RETAKE_UI: input.ui, RETAKE_SESSION: id, RETAKE_OUT: process.env.RETAKE_OUT ?? "", RETAKE_PROJECT: input.project ?? "" };
|
|
60
|
+
const mcpConfig = { mcpServers: { retake: { command: tools.command, args: tools.args, env: toolEnv } } };
|
|
61
|
+
const cfgPath = path.join(input.root, ".drafts", `mcp-${id}.json`);
|
|
62
|
+
fs.mkdirSync(path.dirname(cfgPath), { recursive: true });
|
|
63
|
+
fs.writeFileSync(cfgPath, JSON.stringify(mcpConfig));
|
|
64
|
+
// A clean env: inside another Claude Code session the inherited vars point
|
|
65
|
+
// the child at the wrong auth (see describe.ts); HOME + PATH is enough.
|
|
66
|
+
const env = { HOME: process.env.HOME ?? "", PATH: process.env.PATH ?? "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin" };
|
|
67
|
+
let catalogPath = null;
|
|
68
|
+
let proc;
|
|
69
|
+
if (input.provider === "claude-code") {
|
|
70
|
+
proc = spawn("claude", [
|
|
71
|
+
"-p", "--output-format", "stream-json", "--verbose",
|
|
72
|
+
...(process.env.RETAKE_CLAUDE_MODEL ? ["--model", process.env.RETAKE_CLAUDE_MODEL] : []),
|
|
73
|
+
"--mcp-config", cfgPath, "--strict-mcp-config",
|
|
74
|
+
// The fence: Retake's tools are allowed, the shell and editors are not.
|
|
75
|
+
"--allowedTools", "mcp__retake__*",
|
|
76
|
+
"--disallowedTools", "Bash", "Edit", "Write", "NotebookEdit", "WebFetch", "WebSearch", "Agent", "Task",
|
|
77
|
+
"--max-turns", "60",
|
|
78
|
+
], { env, cwd: input.root, stdio: ["pipe", "pipe", "pipe"] });
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
// Codex installations share ~/.codex/models_cache.json, so a newer app or
|
|
82
|
+
// extension can leave a cache schema an older CLI cannot deserialize. Use
|
|
83
|
+
// the catalog bundled with the exact binary Retake launches instead. The
|
|
84
|
+
// official model_catalog_json override keeps this invocation independent
|
|
85
|
+
// from that mutable shared cache.
|
|
86
|
+
const codex = execFileSync("which", ["codex"], { encoding: "utf8", env }).trim() || "codex";
|
|
87
|
+
catalogPath = path.join(input.root, ".drafts", `codex-models-${id}.json`);
|
|
88
|
+
const catalog = execFileSync(codex, ["debug", "models", "--bundled"], { encoding: "utf8", env, timeout: 15_000, maxBuffer: 4e6 });
|
|
89
|
+
JSON.parse(catalog); // fail before spawning with a corrupt/partial catalog
|
|
90
|
+
fs.writeFileSync(catalogPath, catalog);
|
|
91
|
+
// Codex: MCP servers come from config; pass ours inline via -c and use read-only sandbox (our tools do the writing).
|
|
92
|
+
proc = spawn(codex, [
|
|
93
|
+
"-a", "never", "exec", "--skip-git-repo-check", "--ephemeral", "--ignore-user-config", "--ignore-rules", "-s", "read-only", "--color", "never", "--json",
|
|
94
|
+
"-c", `model_catalog_json=${JSON.stringify(catalogPath)}`,
|
|
95
|
+
...(process.env.RETAKE_CODEX_MODEL ? ["-m", process.env.RETAKE_CODEX_MODEL] : []),
|
|
96
|
+
...(process.env.RETAKE_CODEX_REASONING ? ["-c", `model_reasoning_effort=${JSON.stringify(process.env.RETAKE_CODEX_REASONING)}`] : []),
|
|
97
|
+
"-c", `mcp_servers.retake.command=${JSON.stringify(tools.command)}`,
|
|
98
|
+
"-c", `mcp_servers.retake.args=${JSON.stringify(tools.args)}`,
|
|
99
|
+
"-c", `mcp_servers.retake.env={${Object.entries(toolEnv).map(([key, value]) => `${key}=${JSON.stringify(value)}`).join(",")}}`,
|
|
100
|
+
"-c", "mcp_servers.retake.required=true",
|
|
101
|
+
"-",
|
|
102
|
+
], { env, cwd: input.root, stdio: ["pipe", "pipe", "pipe"] });
|
|
103
|
+
}
|
|
104
|
+
s.proc = proc;
|
|
105
|
+
proc.stdin?.write(brief(input));
|
|
106
|
+
proc.stdin?.end();
|
|
107
|
+
let buf = "";
|
|
108
|
+
proc.stdout?.on("data", (chunk) => {
|
|
109
|
+
buf += chunk.toString();
|
|
110
|
+
let i;
|
|
111
|
+
while ((i = buf.indexOf("\n")) >= 0) {
|
|
112
|
+
const line = buf.slice(0, i).trim();
|
|
113
|
+
buf = buf.slice(i + 1);
|
|
114
|
+
if (!line.startsWith("{"))
|
|
115
|
+
continue;
|
|
116
|
+
try {
|
|
117
|
+
handle(s, JSON.parse(line));
|
|
118
|
+
}
|
|
119
|
+
catch { /* ignore */ }
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
proc.stderr?.on("data", (c) => { const t = c.toString().trim(); if (t && !/SessionEnd hook|hook_/.test(t))
|
|
123
|
+
push(s, "line", `· ${t.slice(0, 200)}`); });
|
|
124
|
+
proc.on("close", (code) => {
|
|
125
|
+
const tail = buf.trim();
|
|
126
|
+
if (tail.startsWith("{")) {
|
|
127
|
+
try {
|
|
128
|
+
handle(s, JSON.parse(tail));
|
|
129
|
+
}
|
|
130
|
+
catch { /* ignore */ }
|
|
131
|
+
}
|
|
132
|
+
s.done = true;
|
|
133
|
+
s.code = code;
|
|
134
|
+
if (!s.demo && s.lastDemo) {
|
|
135
|
+
s.demo = s.lastDemo;
|
|
136
|
+
s.onDemo?.(s.demo);
|
|
137
|
+
}
|
|
138
|
+
if (!s.summary) {
|
|
139
|
+
s.summary = s.lastMessage || (code === 0
|
|
140
|
+
? "The operator finished without creating a demo. Please try the request again."
|
|
141
|
+
: "The operator stopped before the demo was ready. The details above show what failed.");
|
|
142
|
+
}
|
|
143
|
+
// A task that used Retake but never called `done` is incomplete, even if
|
|
144
|
+
// the agent process itself happened to exit cleanly.
|
|
145
|
+
if (s.code === 0 && s.usedTools && !s.demo)
|
|
146
|
+
s.code = 2;
|
|
147
|
+
push(s, "done", { code: s.code, summary: s.summary, demo: s.demo, cost: s.cost, turns: s.turns });
|
|
148
|
+
fs.rmSync(cfgPath, { force: true });
|
|
149
|
+
if (catalogPath)
|
|
150
|
+
fs.rmSync(catalogPath, { force: true });
|
|
151
|
+
});
|
|
152
|
+
return s;
|
|
153
|
+
}
|
|
154
|
+
/** Turn the CLI's event stream into plain lines for the page. */
|
|
155
|
+
function handle(s, d) {
|
|
156
|
+
const type = d.type;
|
|
157
|
+
if (type === "assistant") {
|
|
158
|
+
const content = (d.message?.content) ?? [];
|
|
159
|
+
for (const c of content) {
|
|
160
|
+
if (c.type === "tool_use") {
|
|
161
|
+
const name = String(c.name ?? "").replace(/^mcp__retake__/, "");
|
|
162
|
+
const inp = c.input ?? {};
|
|
163
|
+
const what = name === "dry" || name === "run" || name === "render" || name === "receipts" || name === "read_manifest" ? `${name} ${inp.name ?? ""}${inp.preview === false ? " (final)" : ""}` : name === "scout" ? `scout ${inp.url ?? ""}` : name === "read_project" ? `read ${inp.dir ?? ""}` : name === "edit" ? `edit ${inp.name ?? ""} (${inp.edits?.length ?? 0} change${(inp.edits?.length ?? 0) === 1 ? "" : "s"})` : name;
|
|
164
|
+
push(s, "tool", what);
|
|
165
|
+
}
|
|
166
|
+
else if (c.type === "text" && c.text && c.text.trim()) {
|
|
167
|
+
push(s, "thought", c.text.trim().slice(0, 300));
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
else if (type === "result") {
|
|
172
|
+
s.cost = Number(d.total_cost_usd ?? 0) || undefined;
|
|
173
|
+
s.turns = Number(d.num_turns ?? 0) || undefined;
|
|
174
|
+
if (d.subtype && d.subtype !== "success")
|
|
175
|
+
push(s, "line", `✗ operator stopped: ${d.subtype}`);
|
|
176
|
+
}
|
|
177
|
+
else if (type === "item" || type === "event" || type === "item.started" || type === "item.completed") {
|
|
178
|
+
// Current Codex JSONL uses item.started/item.completed. Keep the older
|
|
179
|
+
// names too so Retake works across CLI upgrades.
|
|
180
|
+
const it = d.item;
|
|
181
|
+
if (it?.type === "mcp_tool_call") {
|
|
182
|
+
s.usedTools = true;
|
|
183
|
+
const name = String(it.tool ?? "tool").replace(/^mcp__retake__/, "");
|
|
184
|
+
const inp = it.arguments ?? {};
|
|
185
|
+
if (type === "item.started" || type === "item") {
|
|
186
|
+
const what = name === "dry" || name === "run" || name === "render" || name === "receipts" || name === "read_manifest"
|
|
187
|
+
? `${name} ${inp.name ?? ""}${inp.preview === false ? " (final)" : ""}`
|
|
188
|
+
: name === "scout" ? `scout ${inp.url ?? ""}`
|
|
189
|
+
: name === "read_project" ? `read ${inp.dir ?? ""}`
|
|
190
|
+
: name === "edit" ? `edit ${inp.name ?? ""} (${inp.edits?.length ?? 0} change${(inp.edits?.length ?? 0) === 1 ? "" : "s"})`
|
|
191
|
+
: name;
|
|
192
|
+
push(s, "tool", what);
|
|
193
|
+
}
|
|
194
|
+
if (type === "item.completed" && it.error?.message)
|
|
195
|
+
push(s, "line", `✗ ${name}: ${it.error.message}`);
|
|
196
|
+
if (type === "item.completed" && name === "run" && it.status === "completed" && typeof inp.name === "string")
|
|
197
|
+
s.lastDemo = inp.name;
|
|
198
|
+
}
|
|
199
|
+
else if (type === "item.completed" && it?.type === "agent_message" && it.text?.trim()) {
|
|
200
|
+
s.lastMessage = it.text.trim();
|
|
201
|
+
push(s, "thought", s.lastMessage.slice(0, 500));
|
|
202
|
+
}
|
|
203
|
+
else if (it?.text) {
|
|
204
|
+
push(s, "thought", it.text.slice(0, 300));
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
else if (type === "turn.failed" || type === "error") {
|
|
208
|
+
const err = d.error;
|
|
209
|
+
const message = typeof err === "string" ? err : err?.message;
|
|
210
|
+
push(s, "line", `✗ ${message || "The operator failed."}`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
// --- pending questions / approvals ------------------------------------------
|
|
214
|
+
export function addPending(s, kind, text, detail) {
|
|
215
|
+
const p = { id: "q" + Date.now().toString(36) + Math.random().toString(36).slice(2, 5), kind, text, detail, answered: false, at: Date.now() };
|
|
216
|
+
s.pending.push(p);
|
|
217
|
+
push(s, "pending", p);
|
|
218
|
+
return p;
|
|
219
|
+
}
|
|
220
|
+
export function answerPending(s, id, answer) {
|
|
221
|
+
const p = s.pending.find((x) => x.id === id);
|
|
222
|
+
if (!p)
|
|
223
|
+
return false;
|
|
224
|
+
p.answered = true;
|
|
225
|
+
p.answer = answer;
|
|
226
|
+
push(s, "answered", { id, answer });
|
|
227
|
+
return true;
|
|
228
|
+
}
|
|
229
|
+
export function markDone(s, summary, demo) { s.summary = summary; s.demo = demo; if (demo)
|
|
230
|
+
s.onDemo?.(demo); push(s, "line", `✓ ${summary}`); }
|
|
231
|
+
export function stopSession(s) { s.proc?.kill("SIGTERM"); }
|