tapline-host 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actionlog.d.ts +43 -0
- package/dist/actionlog.js +62 -0
- package/dist/actionlog.js.map +1 -0
- package/dist/audit.d.ts +44 -0
- package/dist/audit.js +60 -0
- package/dist/audit.js.map +1 -0
- package/dist/cli.js +82 -17
- package/dist/cli.js.map +1 -1
- package/dist/confirm.d.ts +69 -0
- package/dist/confirm.js +120 -0
- package/dist/confirm.js.map +1 -0
- package/dist/format.d.ts +8 -1
- package/dist/format.js +53 -11
- package/dist/format.js.map +1 -1
- package/dist/images.d.ts +7 -0
- package/dist/images.js +10 -0
- package/dist/images.js.map +1 -1
- package/dist/init.d.ts +59 -0
- package/dist/init.js +475 -0
- package/dist/init.js.map +1 -0
- package/dist/inspect.d.ts +75 -0
- package/dist/inspect.js +127 -0
- package/dist/inspect.js.map +1 -0
- package/dist/mcp.d.ts +4 -1
- package/dist/mcp.js +56 -2
- package/dist/mcp.js.map +1 -1
- package/dist/policy.d.ts +42 -0
- package/dist/policy.js +74 -0
- package/dist/policy.js.map +1 -0
- package/dist/run.d.ts +53 -0
- package/dist/run.js +97 -0
- package/dist/run.js.map +1 -0
- package/dist/script.d.ts +76 -0
- package/dist/script.js +169 -0
- package/dist/script.js.map +1 -0
- package/dist/server.d.ts +39 -0
- package/dist/server.js +120 -0
- package/dist/server.js.map +1 -1
- package/dist/tool-text-data.d.ts +25 -8
- package/dist/tool-text-data.js +25 -8
- package/dist/tool-text-data.js.map +1 -1
- package/dist/tool-text.d.ts +1 -1
- package/dist/tools.d.ts +221 -7
- package/dist/tools.js +1451 -168
- package/dist/tools.js.map +1 -1
- package/dist/viewer-page.js +1 -1
- package/dist/viewer-page.js.map +1 -1
- package/dist/viewer.d.ts +3 -0
- package/dist/viewer.js +21 -4
- package/dist/viewer.js.map +1 -1
- package/package.json +2 -2
- package/skills/tapline/SKILL.md +42 -12
- package/skills/tapline/references/setup.md +24 -2
package/dist/tools.js
CHANGED
|
@@ -1,24 +1,31 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { dirname, join } from "node:path";
|
|
1
|
+
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, join } from "node:path";
|
|
3
3
|
import { toBuffer as qrPng } from "qrcode";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { annotateScreenshot } from "./annotate.js";
|
|
6
6
|
import { agentKind, captionFor, personKind, reasonOf } from "./captions.js";
|
|
7
|
-
import { composeFilmstrip, diffImages } from "./images.js";
|
|
7
|
+
import { composeFilmstrip, diffImages, imageSize } from "./images.js";
|
|
8
8
|
import { alertNotes, describeElement, formatChanges, formatElement, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
|
|
9
|
+
import { formatInspect, inspectResultSchema } from "./inspect.js";
|
|
10
|
+
import { AUDIT_TYPES, auditResultSchema, auditTitle, formatAudit } from "./audit.js";
|
|
11
|
+
import { ActionLog, loggedTarget } from "./actionlog.js";
|
|
12
|
+
import { Confirmations } from "./confirm.js";
|
|
13
|
+
import { DEFAULT_POLICY, elementKey, listsFor, matchWord } from "./policy.js";
|
|
14
|
+
import { readScript, refStep, scriptSteps } from "./script.js";
|
|
15
|
+
import { EXPECT_TIMEOUT_S, MAX_STEPS, RUN_TIMEOUT_MS, RUN_TOOLS, describeRunStep, expectFailure, parseError, stepNotes, stepSchema, } from "./run.js";
|
|
9
16
|
import { PAIR_CODE_TTL_MS, forgetLink, pairingLink } from "./pairing.js";
|
|
10
17
|
import { elementSchema, logEntrySchema, screenshotResultSchema, treeResultSchema, } from "./protocol.js";
|
|
11
18
|
import { TaplineError } from "./server.js";
|
|
12
19
|
// ---- input schemas ----------------------------------------------------------
|
|
13
20
|
export const targetSchema = z
|
|
14
21
|
.object({
|
|
15
|
-
ref: z.string().optional().describe('
|
|
16
|
-
id: z.string().optional().describe("
|
|
17
|
-
label: z.string().optional().describe("Accessibility label, exact
|
|
18
|
-
text: z.string().optional().describe("Visible text:
|
|
19
|
-
index: z.number().int().min(0).optional().describe("
|
|
20
|
-
x: z.number().optional().describe("
|
|
21
|
-
y: z.number().optional().describe("
|
|
22
|
+
ref: z.string().optional().describe('From your latest list, e.g. "e12"'),
|
|
23
|
+
id: z.string().optional().describe("The testID, exact"),
|
|
24
|
+
label: z.string().optional().describe("Accessibility label, exact"),
|
|
25
|
+
text: z.string().optional().describe("Visible text: exact first, then part"),
|
|
26
|
+
index: z.number().int().min(0).optional().describe("The Nth match of id, label or text (0-based)"),
|
|
27
|
+
x: z.number().optional().describe("Points, with y"),
|
|
28
|
+
y: z.number().optional().describe("Points, with x"),
|
|
22
29
|
})
|
|
23
30
|
.strict()
|
|
24
31
|
.superRefine((value, ctx) => {
|
|
@@ -40,22 +47,38 @@ export const targetSchema = z
|
|
|
40
47
|
ctx.addIssue({ code: "custom", message: "index works with id, label or text." });
|
|
41
48
|
}
|
|
42
49
|
});
|
|
43
|
-
const
|
|
50
|
+
const rectSchema = z
|
|
51
|
+
.object({
|
|
52
|
+
x: z.number().describe("Points"),
|
|
53
|
+
y: z.number().describe("Points"),
|
|
54
|
+
width: z.number().positive().describe("Points"),
|
|
55
|
+
height: z.number().positive().describe("Points"),
|
|
56
|
+
})
|
|
57
|
+
.strict();
|
|
58
|
+
const ignoreListSchema = z.array(z.union([rectSchema, targetSchema]));
|
|
59
|
+
const targetField = targetSchema.describe("Exactly one of ref, id, label, text or x and y");
|
|
44
60
|
const lookField = z
|
|
45
61
|
.boolean()
|
|
46
62
|
.optional()
|
|
47
|
-
.describe("
|
|
48
|
-
const screenshotField = z.boolean().optional().describe("Add a screenshot
|
|
63
|
+
.describe("false: only the first line; the next answer's changes then cover this action too");
|
|
64
|
+
const screenshotField = z.boolean().optional().describe("Add a screenshot (default false)");
|
|
65
|
+
const dryRunField = z
|
|
66
|
+
.boolean()
|
|
67
|
+
.optional()
|
|
68
|
+
.describe("true: run every check and say what would happen; touch nothing");
|
|
69
|
+
/** `index` written beside `target` rather than in it: agents do that, and it means the same. */
|
|
70
|
+
const indexField = z.number().int().min(0).optional().describe("Same as target.index");
|
|
49
71
|
const forceField = z
|
|
50
72
|
.boolean()
|
|
51
73
|
.optional()
|
|
52
|
-
.describe("Skip the
|
|
74
|
+
.describe("Skip the reach checks (never disabled), only when you've decided a refusal is wrong");
|
|
53
75
|
export const toolShapes = {
|
|
54
76
|
apps: {},
|
|
55
77
|
open_app: { app: z.string().describe("The app's session id, app id or name, as apps lists them") },
|
|
56
78
|
pair: {
|
|
57
79
|
scheme: z
|
|
58
80
|
.string()
|
|
81
|
+
.optional()
|
|
59
82
|
.describe('The app\'s URL scheme ("scheme" in app.json or app.config), e.g. "myapp"; for an app running in Expo Go, the address Expo prints, e.g. "exp://192.168.1.20:8081"'),
|
|
60
83
|
},
|
|
61
84
|
look: {
|
|
@@ -73,9 +96,17 @@ export const toolShapes = {
|
|
|
73
96
|
.describe("Draw each actionable element's box and ref (e12) on the screenshot, so whoever views it can name elements by ref"),
|
|
74
97
|
},
|
|
75
98
|
check_layout: {},
|
|
76
|
-
|
|
99
|
+
audit: {
|
|
100
|
+
types: z
|
|
101
|
+
.array(z.enum(AUDIT_TYPES))
|
|
102
|
+
.optional()
|
|
103
|
+
.describe("Which checks to run (default all): contrast, hit_region, description, dynamic_type, clipped_text, trait, element_detection, overlap, duplicate_id"),
|
|
104
|
+
},
|
|
105
|
+
inspect: { target: targetField },
|
|
106
|
+
tap: { target: targetField, index: indexField, force: forceField, dry_run: dryRunField, look: lookField, screenshot: screenshotField },
|
|
77
107
|
long_press: {
|
|
78
108
|
target: targetField,
|
|
109
|
+
index: indexField,
|
|
79
110
|
seconds: z
|
|
80
111
|
.number()
|
|
81
112
|
.min(0.1)
|
|
@@ -83,15 +114,18 @@ export const toolShapes = {
|
|
|
83
114
|
.optional()
|
|
84
115
|
.describe("How long to hold (default 1.0). Tapline calls the handler, so it doesn't wait"),
|
|
85
116
|
force: forceField,
|
|
117
|
+
dry_run: dryRunField,
|
|
86
118
|
look: lookField,
|
|
87
119
|
screenshot: screenshotField,
|
|
88
120
|
},
|
|
89
121
|
type_text: {
|
|
90
122
|
text: z.string().describe("Text to enter"),
|
|
91
123
|
target: targetField.optional().describe("Input to type into; default: the focused input, else the only input on screen (else ambiguous; none at all is not_found)"),
|
|
124
|
+
index: indexField,
|
|
92
125
|
replace: z.boolean().optional().describe("Replace the current text (default true); false appends"),
|
|
93
126
|
submit: z.boolean().optional().describe("Also submit (like pressing return) after typing (default false)"),
|
|
94
127
|
force: forceField,
|
|
128
|
+
dry_run: dryRunField,
|
|
95
129
|
look: lookField,
|
|
96
130
|
screenshot: screenshotField,
|
|
97
131
|
},
|
|
@@ -105,13 +139,16 @@ export const toolShapes = {
|
|
|
105
139
|
target: targetField
|
|
106
140
|
.optional()
|
|
107
141
|
.describe('Scroll view to scroll, or an element inside it; default: the largest one on screen. With to: "target", the element to bring onto the screen'),
|
|
142
|
+
dry_run: dryRunField,
|
|
108
143
|
look: lookField,
|
|
109
144
|
screenshot: screenshotField,
|
|
110
145
|
},
|
|
111
146
|
toggle: {
|
|
112
147
|
target: targetField,
|
|
148
|
+
index: indexField,
|
|
113
149
|
value: z.boolean().optional().describe("Desired state; omit to flip"),
|
|
114
150
|
force: forceField,
|
|
151
|
+
dry_run: dryRunField,
|
|
115
152
|
look: lookField,
|
|
116
153
|
screenshot: screenshotField,
|
|
117
154
|
},
|
|
@@ -120,13 +157,14 @@ export const toolShapes = {
|
|
|
120
157
|
look: lookField,
|
|
121
158
|
screenshot: screenshotField,
|
|
122
159
|
},
|
|
123
|
-
back: { look: lookField, screenshot: screenshotField },
|
|
160
|
+
back: { dry_run: dryRunField, look: lookField, screenshot: screenshotField },
|
|
124
161
|
swipe: {
|
|
125
162
|
x0: z.number().describe("Where the finger starts, in points from the left"),
|
|
126
163
|
y0: z.number().describe("Where the finger starts, in points from the top"),
|
|
127
164
|
x1: z.number().describe("Where it lifts, in points from the left"),
|
|
128
165
|
y1: z.number().describe("Where it lifts, in points from the top"),
|
|
129
166
|
seconds: z.number().min(0.05).max(5).optional().describe("How long the drag takes (default 0.35)"),
|
|
167
|
+
dry_run: dryRunField,
|
|
130
168
|
look: lookField,
|
|
131
169
|
screenshot: screenshotField,
|
|
132
170
|
},
|
|
@@ -146,6 +184,16 @@ export const toolShapes = {
|
|
|
146
184
|
.max(1320)
|
|
147
185
|
.optional()
|
|
148
186
|
.describe("Screenshot width in pixels, 160 to 1320 (default 600); keep it the same for a name"),
|
|
187
|
+
threshold: z
|
|
188
|
+
.number()
|
|
189
|
+
.min(0)
|
|
190
|
+
.max(100)
|
|
191
|
+
.optional()
|
|
192
|
+
.describe("The share of the screen, in percent, 0 to 100 (default 0.5), that may change and still match"),
|
|
193
|
+
ignore: z
|
|
194
|
+
.array(z.union([rectSchema, targetSchema]))
|
|
195
|
+
.optional()
|
|
196
|
+
.describe("Painted out on both pictures before comparing (a clock, an avatar): targets {id}, {label}, {text}, {ref}, or a rect {x, y, width, height} in points"),
|
|
149
197
|
},
|
|
150
198
|
filmstrip: {
|
|
151
199
|
frames: z.number().int().min(2).max(9).optional().describe("How many frames (default 6)"),
|
|
@@ -155,13 +203,20 @@ export const toolShapes = {
|
|
|
155
203
|
},
|
|
156
204
|
viewer: {},
|
|
157
205
|
record: {
|
|
206
|
+
start: z.boolean().optional().describe("Start recording: every action that succeeds from now on becomes a step"),
|
|
207
|
+
stop: z.boolean().optional().describe("Stop and save the script (to path, else to the scripts folder under name)"),
|
|
208
|
+
name: z.string().optional().describe("The script's name: letters, digits, '.', '_' and '-'"),
|
|
209
|
+
path: z.string().optional().describe("Where stop saves the script file (optional)"),
|
|
158
210
|
action: z
|
|
159
211
|
.enum(["start", "stop", "status", "discard"])
|
|
160
|
-
.
|
|
161
|
-
|
|
212
|
+
.optional()
|
|
213
|
+
.describe("Older form: start, stop, status or discard"),
|
|
162
214
|
},
|
|
163
215
|
replay: {
|
|
164
|
-
name: z.string().optional().describe("
|
|
216
|
+
name: z.string().optional().describe("A saved script's name; omit name and path to list saved scripts"),
|
|
217
|
+
path: z.string().optional().describe("A script file to replay (the shared agent-tools-script format)"),
|
|
218
|
+
look: lookField,
|
|
219
|
+
screenshot: screenshotField,
|
|
165
220
|
},
|
|
166
221
|
wait_for: {
|
|
167
222
|
text: z.string().optional().describe("Wait until this visible text is on screen (or gone)"),
|
|
@@ -178,7 +233,93 @@ export const toolShapes = {
|
|
|
178
233
|
},
|
|
179
234
|
reload: { look: lookField },
|
|
180
235
|
wait: { seconds: z.number().min(0.1).max(30).describe("How long to wait, 0.1 to 30 seconds"), screenshot: screenshotField },
|
|
236
|
+
run: {
|
|
237
|
+
steps: z
|
|
238
|
+
.array(stepSchema)
|
|
239
|
+
.min(1)
|
|
240
|
+
.max(MAX_STEPS)
|
|
241
|
+
.describe('The steps, in order: each is a tool call as an object, {"tool": "tap", "target": {...}, "expect": {...}}. Tools: tap, long_press, type_text, scroll, toggle, back, swipe, open_url, wait, wait_for, app_command, compare_screen (checks, never saves). expect (optional): one of text, id or label, with gone, value and timeout (seconds, default 2); or matches (a reference name, alone or with one of them) with threshold.'),
|
|
242
|
+
trace: z.boolean().optional().describe("Also list each step that ran, one line each"),
|
|
243
|
+
dry_run: z
|
|
244
|
+
.boolean()
|
|
245
|
+
.optional()
|
|
246
|
+
.describe("Check every step's parameters and resolve step 1 (later steps depend on what it does), then touch nothing"),
|
|
247
|
+
look: lookField,
|
|
248
|
+
screenshot: screenshotField,
|
|
249
|
+
},
|
|
181
250
|
};
|
|
251
|
+
// ---- compare_screen ---------------------------------------------------------
|
|
252
|
+
const DEFAULT_MATCH_THRESHOLD = 0.5;
|
|
253
|
+
const MAX_REGION_LINES = 8;
|
|
254
|
+
function describeIgnore(item) {
|
|
255
|
+
if (item.id !== undefined)
|
|
256
|
+
return `id=${item.id}`;
|
|
257
|
+
if (item.label !== undefined)
|
|
258
|
+
return `label ${JSON.stringify(item.label)}`;
|
|
259
|
+
if (item.text !== undefined)
|
|
260
|
+
return JSON.stringify(item.text);
|
|
261
|
+
if (item.ref !== undefined)
|
|
262
|
+
return item.ref;
|
|
263
|
+
return `${String(item.x)},${String(item.y)}`;
|
|
264
|
+
}
|
|
265
|
+
/** The pixel rectangles to paint out for `ignore`, from a fresh element list; targets not on the screen give a note. */
|
|
266
|
+
/**
|
|
267
|
+
* An element as a {label} target matches it: its accessibility label, else its text, as the app's SDK matches
|
|
268
|
+
* (labelOf in the SDK). The ignore masks, recorded indexes and expectations use the same rule as the targets.
|
|
269
|
+
*/
|
|
270
|
+
function labelOfElement(e) {
|
|
271
|
+
return e.label ?? e.text ?? "";
|
|
272
|
+
}
|
|
273
|
+
function ignoreRects(ignore, elements, scale, size) {
|
|
274
|
+
const rects = [];
|
|
275
|
+
const notes = [];
|
|
276
|
+
const add = (x, y, width, height) => {
|
|
277
|
+
const x0 = Math.max(0, Math.floor(x * scale));
|
|
278
|
+
const y0 = Math.max(0, Math.floor(y * scale));
|
|
279
|
+
const x1 = Math.min(size.width, Math.ceil((x + width) * scale));
|
|
280
|
+
const y1 = Math.min(size.height, Math.ceil((y + height) * scale));
|
|
281
|
+
if (x1 > x0 && y1 > y0)
|
|
282
|
+
rects.push({ x: x0, y: y0, width: x1 - x0, height: y1 - y0 });
|
|
283
|
+
};
|
|
284
|
+
for (const item of ignore) {
|
|
285
|
+
if ("width" in item) {
|
|
286
|
+
add(item.x, item.y, item.width, item.height);
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
289
|
+
let found = [];
|
|
290
|
+
if (item.ref !== undefined)
|
|
291
|
+
found = elements.filter((e) => e.ref === item.ref);
|
|
292
|
+
else if (item.id !== undefined)
|
|
293
|
+
found = elements.filter((e) => e.testID === item.id);
|
|
294
|
+
else if (item.label !== undefined) {
|
|
295
|
+
const wanted = item.label.toLowerCase();
|
|
296
|
+
found = elements.filter((e) => labelOfElement(e).toLowerCase() === wanted);
|
|
297
|
+
}
|
|
298
|
+
else if (item.text !== undefined) {
|
|
299
|
+
const wanted = item.text.toLowerCase();
|
|
300
|
+
found = elements.filter((e) => (e.text ?? "").toLowerCase() === wanted);
|
|
301
|
+
if (found.length === 0)
|
|
302
|
+
found = elements.filter((e) => wanted !== "" && (e.text ?? "").toLowerCase().includes(wanted));
|
|
303
|
+
}
|
|
304
|
+
else if (item.x !== undefined && item.y !== undefined) {
|
|
305
|
+
const px = item.x;
|
|
306
|
+
const py = item.y;
|
|
307
|
+
found = elements
|
|
308
|
+
.filter((e) => e.frame && px >= e.frame.x && px <= e.frame.x + e.frame.width && py >= e.frame.y && py <= e.frame.y + e.frame.height)
|
|
309
|
+
.sort((a, b) => (a.frame?.width ?? 0) * (a.frame?.height ?? 0) - (b.frame?.width ?? 0) * (b.frame?.height ?? 0))
|
|
310
|
+
.slice(0, 1);
|
|
311
|
+
}
|
|
312
|
+
found = found.filter((e) => e.frame !== undefined);
|
|
313
|
+
if (item.index !== undefined)
|
|
314
|
+
found = found.slice(item.index, item.index + 1);
|
|
315
|
+
if (found.length === 0)
|
|
316
|
+
notes.push(`ignore ${describeIgnore(item)} isn't on the screen; skipped.`);
|
|
317
|
+
for (const e of found)
|
|
318
|
+
if (e.frame)
|
|
319
|
+
add(e.frame.x, e.frame.y, e.frame.width, e.frame.height);
|
|
320
|
+
}
|
|
321
|
+
return { rects, notes };
|
|
322
|
+
}
|
|
182
323
|
/**
|
|
183
324
|
* Under this many points from its first point, a pointer path is a tap or a hold, not a drag.
|
|
184
325
|
* Tapwright uses the same number, and the viewer page's pending ring (TAP_SLOP in viewer.html)
|
|
@@ -188,6 +329,8 @@ const TAP_SLOP = 10;
|
|
|
188
329
|
/** Held at least this long, a tap is a long press. */
|
|
189
330
|
const LONG_PRESS_MS = 500;
|
|
190
331
|
const SCRIPT_NAME = /^[A-Za-z0-9][\w.-]{0,60}$/;
|
|
332
|
+
/** The answer to a reference name that isn't one: no slashes, so no way out of the references folder. */
|
|
333
|
+
const BAD_REFERENCE_NAME = "Error bad_params: name the reference with letters, digits, '.', '_' and '-'.";
|
|
191
334
|
const scriptSchema = z.object({
|
|
192
335
|
version: z.literal(1),
|
|
193
336
|
name: z.string(),
|
|
@@ -341,6 +484,10 @@ function text(message) {
|
|
|
341
484
|
function fail(message) {
|
|
342
485
|
return { content: [{ type: "text", text: message }], isError: true };
|
|
343
486
|
}
|
|
487
|
+
/** The target with an `index` that was written beside it moved inside (its own index wins). */
|
|
488
|
+
function withIndex(target, index) {
|
|
489
|
+
return index !== undefined && target.index === undefined ? { ...target, index } : target;
|
|
490
|
+
}
|
|
344
491
|
function toWireTarget(target) {
|
|
345
492
|
const withIndex = (t) => target.index !== undefined ? { ...t, index: target.index } : t;
|
|
346
493
|
if (target.ref !== undefined)
|
|
@@ -421,12 +568,75 @@ export function describeError(error) {
|
|
|
421
568
|
}
|
|
422
569
|
return `Error internal: ${error instanceof Error ? error.message : String(error)}`;
|
|
423
570
|
}
|
|
571
|
+
/** Ends a guarded call inside the agent's turn with the policy's answer (nothing was touched). */
|
|
572
|
+
class PolicyStop extends Error {
|
|
573
|
+
answer;
|
|
574
|
+
constructor(answer) {
|
|
575
|
+
super("stopped by the policy");
|
|
576
|
+
this.answer = answer;
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
/** Adds a note line to the end of an answer's first text block. */
|
|
580
|
+
function appendNote(answer, note) {
|
|
581
|
+
const [top, ...rest] = answer.content;
|
|
582
|
+
if (top === undefined || top.type !== "text")
|
|
583
|
+
return answer;
|
|
584
|
+
return { ...answer, content: [{ type: "text", text: [top.text.trimEnd(), note].join("\n") }, ...rest] };
|
|
585
|
+
}
|
|
586
|
+
/** An app's dry-run answer (AGENT-TOOLS 1.8 §11.4). */
|
|
587
|
+
const dryRunResultSchema = z.object({
|
|
588
|
+
element: elementSchema.nullish(),
|
|
589
|
+
dryRun: z.literal(true),
|
|
590
|
+
point: z.object({ x: z.number(), y: z.number() }).optional(),
|
|
591
|
+
value: z.unknown().optional(),
|
|
592
|
+
});
|
|
593
|
+
/** For an app without dry runs: every element on its list the target could mean. */
|
|
594
|
+
function candidatesFor(target, elements) {
|
|
595
|
+
if (typeof target !== "object" || target === null)
|
|
596
|
+
return [];
|
|
597
|
+
const t = target;
|
|
598
|
+
const lower = (value) => (typeof value === "string" ? value.toLowerCase() : "");
|
|
599
|
+
const x = t["x"];
|
|
600
|
+
const y = t["y"];
|
|
601
|
+
let found;
|
|
602
|
+
if (typeof t["ref"] === "string")
|
|
603
|
+
found = elements.filter((e) => e.ref === t["ref"]);
|
|
604
|
+
else if (typeof t["testID"] === "string")
|
|
605
|
+
found = elements.filter((e) => e.testID === t["testID"]);
|
|
606
|
+
else if (typeof t["label"] === "string")
|
|
607
|
+
found = elements.filter((e) => lower(e.label) === lower(t["label"]));
|
|
608
|
+
else if (typeof t["text"] === "string") {
|
|
609
|
+
const want = lower(t["text"]);
|
|
610
|
+
found = elements.filter((e) => lower(e.text).includes(want) || lower(e.label).includes(want));
|
|
611
|
+
}
|
|
612
|
+
else if (typeof x === "number" && typeof y === "number") {
|
|
613
|
+
found = elements.filter((e) => {
|
|
614
|
+
const f = e.frame;
|
|
615
|
+
return f !== null && f !== undefined && x >= f.x && x <= f.x + f.width && y >= f.y && y <= f.y + f.height;
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
else
|
|
619
|
+
found = [];
|
|
620
|
+
const index = t["index"];
|
|
621
|
+
const picked = typeof index === "number" ? found[index] : undefined;
|
|
622
|
+
return picked !== undefined ? [picked] : found;
|
|
623
|
+
}
|
|
424
624
|
// ---- tool handlers ----------------------------------------------------------
|
|
425
625
|
/** The tool handlers behind the MCP server. They only talk to a TaplineHost, so they are easy to test. */
|
|
426
626
|
export class TaplineTools {
|
|
427
627
|
host;
|
|
428
628
|
options;
|
|
429
629
|
reloadWaitMs;
|
|
630
|
+
/** The action log (§12), when one is configured. */
|
|
631
|
+
actionLog;
|
|
632
|
+
/** When the action in flight started, for the log's `ms`. */
|
|
633
|
+
actionStartedAt;
|
|
634
|
+
/** Set while `run` runs a step: the log ties the action to its batch. */
|
|
635
|
+
batch;
|
|
636
|
+
/** Extra fields for the next log line (a dry run, a confirmed touch). */
|
|
637
|
+
logExtra;
|
|
638
|
+
confirmations;
|
|
639
|
+
policy;
|
|
430
640
|
reloadSettleMs;
|
|
431
641
|
noScreenshot = new Set();
|
|
432
642
|
shotFiles = [];
|
|
@@ -444,6 +654,15 @@ export class TaplineTools {
|
|
|
444
654
|
personNotePending = false;
|
|
445
655
|
recording = null;
|
|
446
656
|
replaying = false;
|
|
657
|
+
/**
|
|
658
|
+
* While a script replays (AGENT-TOOLS 1.9 §14.3): the references folder next to it (looked in first), where a
|
|
659
|
+
* failed check puts its red picture, and whether the checks save instead of compare (`--update-references`).
|
|
660
|
+
*/
|
|
661
|
+
replayContext = null;
|
|
662
|
+
/** Where the latest failed check of the running batch saved its picture. */
|
|
663
|
+
diffSaved = null;
|
|
664
|
+
/** The elements the latest comparison painted its ignore list from: a recording saves its mask from the same list. */
|
|
665
|
+
lastMaskElements;
|
|
447
666
|
/** The screen size in points from the latest list or overlay, for the live viewer's 0..1 positions. */
|
|
448
667
|
lastScreen;
|
|
449
668
|
shotCount = 0;
|
|
@@ -452,6 +671,10 @@ export class TaplineTools {
|
|
|
452
671
|
this.host = host;
|
|
453
672
|
this.options = options;
|
|
454
673
|
this.reloadWaitMs = options.reloadWaitMs ?? 20_000;
|
|
674
|
+
this.confirmations = options.confirmations ?? new Confirmations();
|
|
675
|
+
this.policy = options.policy ?? DEFAULT_POLICY;
|
|
676
|
+
this.actionLog = options.logDir !== undefined ? new ActionLog(options.logDir) : null;
|
|
677
|
+
this.actionLog?.open();
|
|
455
678
|
this.reloadSettleMs = options.reloadSettleMs ?? 1500;
|
|
456
679
|
// A finger on the phone itself: the app saw a real touch. It can't be queued like viewer input
|
|
457
680
|
// (it has already happened), so it only leaves the note and makes `{ref}` targets stale.
|
|
@@ -490,15 +713,17 @@ export class TaplineTools {
|
|
|
490
713
|
const silent = this.silentDropMessage();
|
|
491
714
|
lines.push(silent !== undefined
|
|
492
715
|
? silent
|
|
493
|
-
: state === "
|
|
494
|
-
? `${last.session.app.name} disconnected at ${when} after
|
|
495
|
-
:
|
|
716
|
+
: state === "inactive"
|
|
717
|
+
? `${last.session.app.name} disconnected at ${when} after iOS put something in front of it (a system prompt such as a password request, a call, or Control Center). Tapline can't see or answer it: ask the user to deal with it on the phone. The app reconnects by itself.`
|
|
718
|
+
: state === "background"
|
|
719
|
+
? `${last.session.app.name} disconnected at ${when} after it went to the background (the user switched apps or locked the phone, and iOS paused it). Ask the user to open the app again: it reconnects by itself.`
|
|
720
|
+
: `${last.session.app.name} disconnected at ${when} (${last.reason}). If the app is still open, it reconnects by itself.`);
|
|
496
721
|
}
|
|
497
722
|
return [
|
|
498
723
|
"No app is connected to Tapline.",
|
|
499
724
|
...lines,
|
|
500
725
|
`This host listens at ${url} on ${name}.`,
|
|
501
|
-
'To connect a phone, call `pair
|
|
726
|
+
'To connect a phone, call `pair`. A development build loading its code from this computer then asks by itself: the user taps Connect. Other builds: pass the app\'s URL scheme ("scheme" in app.json, e.g. "myapp") and show the user the QR code to scan.',
|
|
502
727
|
"The phone needs to be on the same Wi-Fi as this computer, and the app must be a build with Tapline turned on, open in the foreground.",
|
|
503
728
|
"A phone that paired with this host before reconnects by itself while the app is open; one paired with a host on another port (e.g. before a restart) needs `pair` again. Once it connects, retry.",
|
|
504
729
|
].join("\n");
|
|
@@ -506,6 +731,19 @@ export class TaplineTools {
|
|
|
506
731
|
// -- pair
|
|
507
732
|
async pair(args) {
|
|
508
733
|
const { url, name } = this.options.connect;
|
|
734
|
+
// Zero-QR pairing (Tapline 1.1): a development build whose JavaScript comes from this computer finds the
|
|
735
|
+
// offer itself and asks its person to connect. Open for 10 minutes, one use, local network only.
|
|
736
|
+
const offer = this.host.openDevOffer(name);
|
|
737
|
+
this.writeOffer(offer);
|
|
738
|
+
const devLine = "A development build that loads its JavaScript from this computer (Metro) on the same Wi-Fi asks by itself within a few seconds: the user only taps Connect, no QR code (open for 10 minutes, once).";
|
|
739
|
+
if (args.scheme === undefined || args.scheme.trim() === "") {
|
|
740
|
+
return text([
|
|
741
|
+
`Pairing is open for development builds for 10 minutes (${name}).`,
|
|
742
|
+
devLine,
|
|
743
|
+
"Ask the user to open the app on the phone and tap Connect, then call `apps`.",
|
|
744
|
+
"For a TestFlight or release build, call pair again with the app's URL scheme to get a QR code.",
|
|
745
|
+
].join("\n"));
|
|
746
|
+
}
|
|
509
747
|
let link;
|
|
510
748
|
let forget;
|
|
511
749
|
try {
|
|
@@ -540,6 +778,7 @@ export class TaplineTools {
|
|
|
540
778
|
"",
|
|
541
779
|
`Show the user the QR code (attached).${saved} They scan it with the phone's camera, open the app and tap Connect.`,
|
|
542
780
|
"The phone needs to be on the same Wi-Fi as this computer, and the app must be a build with Tapline turned on.",
|
|
781
|
+
devLine,
|
|
543
782
|
"Once paired, the app reconnects to this host (this address and port) by itself whenever it is open; call `apps` to check.",
|
|
544
783
|
`To unpair the phone, open ${forget} on it.`,
|
|
545
784
|
];
|
|
@@ -550,6 +789,51 @@ export class TaplineTools {
|
|
|
550
789
|
],
|
|
551
790
|
};
|
|
552
791
|
}
|
|
792
|
+
/** `<offerDir>/<port>.json` (0600): the open offer for the Metro helper. Never fails the pair call. */
|
|
793
|
+
writeOffer(offer) {
|
|
794
|
+
const dir = this.options.offerDir;
|
|
795
|
+
const port = this.host.boundPort();
|
|
796
|
+
if (dir === undefined || port === undefined)
|
|
797
|
+
return;
|
|
798
|
+
try {
|
|
799
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
800
|
+
const file = join(dir, `${port}.json`);
|
|
801
|
+
writeFileSync(file, JSON.stringify({ code: offer.code, name: offer.name, port, until: offer.until }), { mode: 0o600 });
|
|
802
|
+
this.host.setDevOfferFile(file);
|
|
803
|
+
const remove = () => {
|
|
804
|
+
try {
|
|
805
|
+
// only our own offer: a newer pair call has written its own
|
|
806
|
+
if (existsSync(file) && readFileSync(file, "utf8").includes(offer.code))
|
|
807
|
+
rmSync(file, { force: true });
|
|
808
|
+
}
|
|
809
|
+
catch {
|
|
810
|
+
// gone already
|
|
811
|
+
}
|
|
812
|
+
};
|
|
813
|
+
// Used: the code works once, so Metro stops handing it out at once.
|
|
814
|
+
const onConnect = () => {
|
|
815
|
+
if (this.host.currentDevOffer()?.code === offer.code)
|
|
816
|
+
return;
|
|
817
|
+
this.host.off("connect", onConnect);
|
|
818
|
+
remove();
|
|
819
|
+
};
|
|
820
|
+
this.host.on("connect", onConnect);
|
|
821
|
+
setTimeout(() => {
|
|
822
|
+
this.host.off("connect", onConnect);
|
|
823
|
+
try {
|
|
824
|
+
// only our own offer: a newer pair call has written its own
|
|
825
|
+
if (existsSync(file) && readFileSync(file, "utf8").includes(offer.code))
|
|
826
|
+
rmSync(file, { force: true });
|
|
827
|
+
}
|
|
828
|
+
catch {
|
|
829
|
+
// gone already
|
|
830
|
+
}
|
|
831
|
+
}, Math.max(0, offer.until - Date.now())).unref();
|
|
832
|
+
}
|
|
833
|
+
catch {
|
|
834
|
+
// the host's own endpoint still has it
|
|
835
|
+
}
|
|
836
|
+
}
|
|
553
837
|
// -- apps
|
|
554
838
|
apps() {
|
|
555
839
|
const sessions = this.host.sessions();
|
|
@@ -602,6 +886,258 @@ export class TaplineTools {
|
|
|
602
886
|
const parts = await this.lookParts(session, args.screenshot ?? true, args.width ?? DEFAULT_SCREENSHOT_WIDTH, false, args.labels === true);
|
|
603
887
|
return { content: parts.content, ...(parts.isError ? { isError: true } : {}) };
|
|
604
888
|
}
|
|
889
|
+
// -- run (AGENT-TOOLS 1.8 §9): a batch of steps, answered once
|
|
890
|
+
async run(args) {
|
|
891
|
+
const session = this.host.current();
|
|
892
|
+
if (session === undefined)
|
|
893
|
+
return this.notConnected();
|
|
894
|
+
const steps = args.steps;
|
|
895
|
+
for (const [i, step] of steps.entries()) {
|
|
896
|
+
if (!RUN_TOOLS.includes(step.tool)) {
|
|
897
|
+
return fail(`Error bad_params: step ${i + 1}: "${step.tool}" can't be a step (steps act; use it on its own call). Steps can be: ${RUN_TOOLS.join(", ")}.`);
|
|
898
|
+
}
|
|
899
|
+
const shape = toolShapes[step.tool];
|
|
900
|
+
const { tool: _tool, expect: _expect, route: _route, ...params } = step;
|
|
901
|
+
if (step.tool === "compare_screen" && params.update !== undefined) {
|
|
902
|
+
return fail(`Error bad_params: step ${i + 1}: update can't be used in run (a step never saves a reference)`);
|
|
903
|
+
}
|
|
904
|
+
const parsed = z.object(shape).strict().safeParse(params);
|
|
905
|
+
if (!parsed.success) {
|
|
906
|
+
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): ${parsed.error.issues.map((issue) => `${issue.path.join(".") || "params"}: ${issue.message}`).join("; ")}`);
|
|
907
|
+
}
|
|
908
|
+
// Every reference name is checked before any step runs: a name with a path in it never reaches the disk.
|
|
909
|
+
const names = [step.tool === "compare_screen" ? params.name : undefined, step.expect?.matches];
|
|
910
|
+
if (names.some((name) => typeof name === "string" && !SCRIPT_NAME.test(name.trim()))) {
|
|
911
|
+
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): name the reference with letters, digits, '.', '_' and '-'.`);
|
|
912
|
+
}
|
|
913
|
+
if (step.expect?.ignore !== undefined) {
|
|
914
|
+
const ignored = ignoreListSchema.safeParse(step.expect.ignore);
|
|
915
|
+
if (!ignored.success) {
|
|
916
|
+
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): expect.ignore: ${ignored.error.issues.map((issue) => issue.message).join("; ")}`);
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
this.diffSaved = null;
|
|
921
|
+
if (args.dry_run === true)
|
|
922
|
+
return this.dryRunBatch(steps);
|
|
923
|
+
const before = this.unlistedActions.get(session.id) ?? 0;
|
|
924
|
+
const started = Date.now();
|
|
925
|
+
const batchId = `b${started.toString(36)}`;
|
|
926
|
+
const notes = [];
|
|
927
|
+
const trace = [];
|
|
928
|
+
let ran = 0;
|
|
929
|
+
let stop = null;
|
|
930
|
+
for (const [i, step] of steps.entries()) {
|
|
931
|
+
if (Date.now() - started > RUN_TIMEOUT_MS) {
|
|
932
|
+
stop = { index: i, code: "timeout", message: `the batch ran longer than ${RUN_TIMEOUT_MS / 1000} s`, rest: [] };
|
|
933
|
+
break;
|
|
934
|
+
}
|
|
935
|
+
const at = Date.now();
|
|
936
|
+
this.batch = { id: batchId, step: i + 1 };
|
|
937
|
+
const answer = await this.runStep(step).finally(() => {
|
|
938
|
+
this.batch = undefined;
|
|
939
|
+
});
|
|
940
|
+
const said = answer.content.map((block) => (block.type === "text" ? block.text : "")).join("\n").trim();
|
|
941
|
+
if (answer.isError === true) {
|
|
942
|
+
stop = { index: i, ...parseError(said) };
|
|
943
|
+
break;
|
|
944
|
+
}
|
|
945
|
+
notes.push(...stepNotes(said, i + 1));
|
|
946
|
+
if (step.expect !== undefined) {
|
|
947
|
+
const missed = await this.expectMissed(step.expect);
|
|
948
|
+
if (missed !== null) {
|
|
949
|
+
stop = { index: i, code: "expect_failed", message: `its action ran, but ${missed}`, rest: [] };
|
|
950
|
+
break;
|
|
951
|
+
}
|
|
952
|
+
// A picture check that held is kept in a recording, as compare_screen's is: right after the action, the
|
|
953
|
+
// script makes it that step's `matches` again (AGENT-TOOLS 1.9 §14.2).
|
|
954
|
+
if (step.expect.matches !== undefined) {
|
|
955
|
+
const recorded = await this.recordCompare({
|
|
956
|
+
name: step.expect.matches,
|
|
957
|
+
...(step.expect.threshold !== undefined ? { threshold: step.expect.threshold } : {}),
|
|
958
|
+
...(step.expect.ignore !== undefined ? { ignore: ignoreListSchema.parse(step.expect.ignore) } : {}),
|
|
959
|
+
});
|
|
960
|
+
if (recorded !== null) {
|
|
961
|
+
stop = { index: i, ...parseError(recorded) };
|
|
962
|
+
break;
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
const route = step.route;
|
|
967
|
+
if (Array.isArray(route) && route.every((part) => typeof part === "string")) {
|
|
968
|
+
const now = await this.currentRoute();
|
|
969
|
+
if (now !== null && now.join(" \u203a ") !== route.join(" \u203a ")) {
|
|
970
|
+
stop = { index: i, code: "expect_failed", message: `its action ran, but it ended on ${now.join(" \u203a ")}; the recording ended on ${route.join(" \u203a ")}`, rest: [] };
|
|
971
|
+
break;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
ran++;
|
|
975
|
+
if (args.trace === true)
|
|
976
|
+
trace.push(` ${i + 1}. ${said.split("\n")[0] ?? ""} (${((Date.now() - at) / 1000).toFixed(1)} s)`);
|
|
977
|
+
}
|
|
978
|
+
// The batch is one call: the changes below are measured from the list the agent had before it.
|
|
979
|
+
if (before > 0)
|
|
980
|
+
this.unlistedActions.set(session.id, before);
|
|
981
|
+
else
|
|
982
|
+
this.unlistedActions.delete(session.id);
|
|
983
|
+
const total = steps.length;
|
|
984
|
+
const failed = stop !== null;
|
|
985
|
+
const head = stop === null
|
|
986
|
+
? `Ran ${total} of ${total} steps in ${((Date.now() - started) / 1000).toFixed(1)} s.`
|
|
987
|
+
: `Error ${stop.code}: Ran ${ran} of ${total} steps; stopped at step ${stop.index + 1} (${describeRunStep(steps[stop.index])}): ${stop.message}`;
|
|
988
|
+
const diffNote = stop !== null && this.diffSaved !== null ? [`Note: the difference is saved at ${this.diffSaved}.`] : [];
|
|
989
|
+
const first = [head, ...trace, ...(stop?.rest ?? []), ...diffNote].join("\n");
|
|
990
|
+
let response;
|
|
991
|
+
const live = this.host.current();
|
|
992
|
+
if (args.look === false || live === undefined) {
|
|
993
|
+
response = text([first, ...notes].join("\n"));
|
|
994
|
+
}
|
|
995
|
+
else {
|
|
996
|
+
response = await this.withLook(live, first, args.screenshot === true);
|
|
997
|
+
if (notes.length > 0) {
|
|
998
|
+
const [top, ...rest] = response.content;
|
|
999
|
+
if (top !== undefined && top.type === "text")
|
|
1000
|
+
response = { ...response, content: [{ type: "text", text: [top.text.trimEnd(), ...notes].join("\n") }, ...rest] };
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
return failed ? { ...response, isError: true } : response;
|
|
1004
|
+
}
|
|
1005
|
+
/** `run` with dry_run (§11.4): the steps are checked already; step 1 is resolved, nothing is touched. */
|
|
1006
|
+
async dryRunBatch(steps) {
|
|
1007
|
+
const total = steps.length;
|
|
1008
|
+
const first = steps[0];
|
|
1009
|
+
const head = `Would run ${total} step${total === 1 ? "" : "s"}`;
|
|
1010
|
+
if (["tap", "long_press", "toggle", "type_text", "scroll", "back", "swipe"].includes(first.tool)) {
|
|
1011
|
+
const answer = await this.runStep({ ...first, dry_run: true });
|
|
1012
|
+
const said = answer.content.map((block) => (block.type === "text" ? block.text : "")).join("\n").trim();
|
|
1013
|
+
const [line, ...rest] = said.split("\n");
|
|
1014
|
+
if (answer.isError === true) {
|
|
1015
|
+
const { code, message } = parseError(said);
|
|
1016
|
+
return fail(`Error ${code}: ${head}; step 1 (${describeRunStep(first)}) would fail: ${message}`);
|
|
1017
|
+
}
|
|
1018
|
+
const would = (line ?? "").replace(/^Would /, "would ");
|
|
1019
|
+
return text([`${head}; step 1 ${would}`, ...rest].join("\n"));
|
|
1020
|
+
}
|
|
1021
|
+
return text(`${head}; step 1 would ${describeRunStep(first)}.`);
|
|
1022
|
+
}
|
|
1023
|
+
/** One step of `run`, as its own tool call without a list. */
|
|
1024
|
+
runStep(step) {
|
|
1025
|
+
const { tool, expect: _expect, route: _route, ...rest } = step;
|
|
1026
|
+
const quiet = { ...rest, look: false, screenshot: false };
|
|
1027
|
+
switch (tool) {
|
|
1028
|
+
case "tap":
|
|
1029
|
+
return this.tap(quiet);
|
|
1030
|
+
case "long_press":
|
|
1031
|
+
return this.longPress(quiet);
|
|
1032
|
+
case "type_text":
|
|
1033
|
+
return this.typeText(quiet);
|
|
1034
|
+
case "scroll":
|
|
1035
|
+
return this.scroll(quiet);
|
|
1036
|
+
case "toggle":
|
|
1037
|
+
return this.toggle(quiet);
|
|
1038
|
+
case "back":
|
|
1039
|
+
return this.back(quiet);
|
|
1040
|
+
case "swipe":
|
|
1041
|
+
return this.swipe(quiet);
|
|
1042
|
+
case "open_url":
|
|
1043
|
+
return this.openUrl(quiet);
|
|
1044
|
+
case "wait":
|
|
1045
|
+
return this.wait({ seconds: Number(rest.seconds ?? 0) });
|
|
1046
|
+
case "wait_for":
|
|
1047
|
+
return this.waitFor(quiet);
|
|
1048
|
+
case "app_command":
|
|
1049
|
+
return this.appCommand(quiet);
|
|
1050
|
+
case "compare_screen":
|
|
1051
|
+
return this.compareStep(rest);
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
/** The screen's route names now (one cheap read), or null when the app has no router or didn't answer. */
|
|
1055
|
+
async currentRoute() {
|
|
1056
|
+
const session = this.host.current();
|
|
1057
|
+
if (session === undefined)
|
|
1058
|
+
return null;
|
|
1059
|
+
const tree = treeResultSchema.safeParse(await this.host.request("tree", { peek: true }, { sessionId: session.id }).catch(() => null));
|
|
1060
|
+
return tree.success && tree.data.route !== undefined ? tree.data.route.names : null;
|
|
1061
|
+
}
|
|
1062
|
+
/** Null when the expectation holds within its time; else how it failed. */
|
|
1063
|
+
async expectMissed(e) {
|
|
1064
|
+
if (e.text !== undefined || e.id !== undefined || e.label !== undefined) {
|
|
1065
|
+
const missed = await this.textExpectMissed(e);
|
|
1066
|
+
if (missed !== null)
|
|
1067
|
+
return missed;
|
|
1068
|
+
}
|
|
1069
|
+
// A picture check comes last, once the rest held.
|
|
1070
|
+
return e.matches === undefined ? null : this.matchMissed(e.matches, e.threshold ?? DEFAULT_MATCH_THRESHOLD, ignoreListSchema.parse(e.ignore ?? []));
|
|
1071
|
+
}
|
|
1072
|
+
async textExpectMissed(e) {
|
|
1073
|
+
const seconds = e.timeout ?? EXPECT_TIMEOUT_S;
|
|
1074
|
+
const which = e.text !== undefined ? { text: e.text } : e.id !== undefined ? { id: e.id } : { label: e.label };
|
|
1075
|
+
const seen = await this.waitFor({ ...which, ...(e.gone === true ? { gone: true } : {}), timeout: Math.max(0.5, seconds), look: false });
|
|
1076
|
+
if (seen.isError === true)
|
|
1077
|
+
return expectFailure(e, seconds);
|
|
1078
|
+
if (e.value === undefined || e.gone === true)
|
|
1079
|
+
return null;
|
|
1080
|
+
// The element is there: wait for its value (a switch reads "on" or "off").
|
|
1081
|
+
const session = this.host.current();
|
|
1082
|
+
if (session === undefined)
|
|
1083
|
+
return expectFailure(e, seconds);
|
|
1084
|
+
const until = Date.now() + seconds * 1000;
|
|
1085
|
+
let actual;
|
|
1086
|
+
for (;;) {
|
|
1087
|
+
const tree = treeResultSchema.safeParse(await this.host.request("tree", { peek: true }, { sessionId: session.id }).catch(() => null));
|
|
1088
|
+
const element = tree.success
|
|
1089
|
+
? tree.data.elements.find((el) => (e.id !== undefined ? el.testID === e.id : labelOfElement(el).toLowerCase() === (e.label ?? "").toLowerCase()))
|
|
1090
|
+
: undefined;
|
|
1091
|
+
if (element !== undefined) {
|
|
1092
|
+
actual = typeof element.checked === "boolean" ? (element.checked ? "on" : "off") : typeof element.value === "string" ? element.value : undefined;
|
|
1093
|
+
// A switch takes "true" and "false" too (it reads checked=true in the list).
|
|
1094
|
+
const wanted = typeof element.checked === "boolean" && (e.value === "true" || e.value === "false") ? (e.value === "true" ? "on" : "off") : e.value;
|
|
1095
|
+
if (actual === wanted)
|
|
1096
|
+
return null;
|
|
1097
|
+
}
|
|
1098
|
+
if (Date.now() >= until)
|
|
1099
|
+
return expectFailure(e, seconds, actual);
|
|
1100
|
+
await sleep(100);
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
// -- inspect (Tapline 1.1): from the screen to the code
|
|
1104
|
+
async inspect(args) {
|
|
1105
|
+
const session = this.host.current();
|
|
1106
|
+
if (session === undefined)
|
|
1107
|
+
return this.notConnected();
|
|
1108
|
+
let raw;
|
|
1109
|
+
try {
|
|
1110
|
+
raw = await this.host.request("inspect", { target: toWireTarget(args.target) }, { sessionId: session.id });
|
|
1111
|
+
}
|
|
1112
|
+
catch (error) {
|
|
1113
|
+
return fail(describeError(error));
|
|
1114
|
+
}
|
|
1115
|
+
const parsed = inspectResultSchema.safeParse(raw);
|
|
1116
|
+
if (!parsed.success)
|
|
1117
|
+
return fail("Error internal: the app returned an inspect answer the host could not read.");
|
|
1118
|
+
return text(formatInspect(parsed.data));
|
|
1119
|
+
}
|
|
1120
|
+
// -- audit (AGENT-TOOLS 1.8 §10)
|
|
1121
|
+
async audit(args = {}) {
|
|
1122
|
+
const session = this.host.current();
|
|
1123
|
+
if (session === undefined)
|
|
1124
|
+
return this.notConnected();
|
|
1125
|
+
if (!session.capabilities.includes("audit")) {
|
|
1126
|
+
return fail("Error unsupported: this app's Tapline is too old for audit; update react-native-tapline (check_layout works meanwhile).");
|
|
1127
|
+
}
|
|
1128
|
+
let raw;
|
|
1129
|
+
try {
|
|
1130
|
+
raw = await this.host.request("audit", args.types !== undefined ? { types: args.types } : {}, { sessionId: session.id });
|
|
1131
|
+
}
|
|
1132
|
+
catch (error) {
|
|
1133
|
+
return fail(describeError(error));
|
|
1134
|
+
}
|
|
1135
|
+
const parsed = auditResultSchema.safeParse(raw);
|
|
1136
|
+
if (!parsed.success)
|
|
1137
|
+
return fail("Error internal: the app returned an audit the host could not read.");
|
|
1138
|
+
const title = auditTitle(await this.currentRoute(), session.app.name);
|
|
1139
|
+
return text([formatAudit(title, parsed.data, args.types), ...this.takePersonNote()].join("\n"));
|
|
1140
|
+
}
|
|
605
1141
|
// -- check_layout
|
|
606
1142
|
async checkLayout(_args = {}) {
|
|
607
1143
|
const session = this.host.current();
|
|
@@ -754,32 +1290,42 @@ export class TaplineTools {
|
|
|
754
1290
|
/** Runs a request, then (unless disabled) appends the fresh element list and how long the app took to settle. */
|
|
755
1291
|
async act(method, params, describe, withLook, screenshot, extraNotes,
|
|
756
1292
|
/** True when the result means nothing was done: the first line is then the whole answer. */
|
|
757
|
-
nothingDone
|
|
1293
|
+
nothingDone,
|
|
1294
|
+
/** Runs inside the agent's turn, right before the request: an answer stops the call there (the policy). */
|
|
1295
|
+
preflight) {
|
|
758
1296
|
const session = this.host.current();
|
|
759
1297
|
if (session === undefined)
|
|
760
1298
|
return this.notConnected();
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
// The error says it already, so the note isn't repeated.
|
|
765
|
-
this.personNotePending = false;
|
|
766
|
-
return fail("Error stale: a person used the app since your last look, so refs from that list may point elsewhere. Look again, or target by id, label or text.");
|
|
767
|
-
}
|
|
768
|
-
}
|
|
1299
|
+
const stale = this.staleRef(params);
|
|
1300
|
+
if (stale !== null)
|
|
1301
|
+
return stale;
|
|
769
1302
|
let result;
|
|
1303
|
+
this.actionStartedAt = Date.now();
|
|
770
1304
|
try {
|
|
771
1305
|
const timeoutMs = answerTimeoutMs(method, params);
|
|
772
|
-
const run = () =>
|
|
1306
|
+
const run = async () => {
|
|
1307
|
+
if (preflight !== undefined) {
|
|
1308
|
+
const stop = await preflight();
|
|
1309
|
+
if (stop !== null)
|
|
1310
|
+
throw new PolicyStop(stop);
|
|
1311
|
+
}
|
|
1312
|
+
return this.host.request(method, params, { sessionId: session.id, ...(timeoutMs !== undefined ? { timeoutMs } : {}) });
|
|
1313
|
+
};
|
|
773
1314
|
// With the live viewer, a person watching goes first.
|
|
774
1315
|
result = this.options.control ? await this.options.control.runAgent(run) : await run();
|
|
775
1316
|
}
|
|
776
1317
|
catch (error) {
|
|
1318
|
+
if (error instanceof PolicyStop)
|
|
1319
|
+
return error.answer;
|
|
1320
|
+
this.logExtra = undefined;
|
|
777
1321
|
this.showAction("agent", method, undefined, params, error instanceof TaplineError ? error.code : "internal", error instanceof TaplineError ? reasonOf(error.code, error.message) : undefined);
|
|
778
1322
|
return fail(describeError(error));
|
|
779
1323
|
}
|
|
780
1324
|
const line = describe(result);
|
|
781
|
-
if (nothingDone?.(result) === true)
|
|
1325
|
+
if (nothingDone?.(result) === true) {
|
|
1326
|
+
this.logExtra = undefined;
|
|
782
1327
|
return text([line, ...this.takePersonNote()].join("\n"));
|
|
1328
|
+
}
|
|
783
1329
|
this.showAction("agent", method, result, params);
|
|
784
1330
|
let response;
|
|
785
1331
|
this.lastLookRoute = undefined;
|
|
@@ -826,8 +1372,14 @@ export class TaplineTools {
|
|
|
826
1372
|
}
|
|
827
1373
|
}
|
|
828
1374
|
if (this.recording !== null && !this.replaying) {
|
|
829
|
-
const { params:
|
|
830
|
-
|
|
1375
|
+
const { params: chosen, fragile } = stableParams(params, result);
|
|
1376
|
+
// A ref saved as a label or text that several elements share needs its index (AGENT-TOOLS 1.8 §9.4).
|
|
1377
|
+
const stable = await this.withIndexIfShared(params, chosen, result, session.id);
|
|
1378
|
+
// A password field's text is never saved in a script (Tapwright's review, M4).
|
|
1379
|
+
const secret = method === "type" && typeof result === "object" && result !== null && "secure" in result && result.secure === true;
|
|
1380
|
+
const step = { method, params: secret ? { ...stable, text: "" } : stable };
|
|
1381
|
+
if (secret)
|
|
1382
|
+
step.secret = true;
|
|
831
1383
|
if (this.lastLookRoute !== undefined)
|
|
832
1384
|
step.route = this.lastLookRoute;
|
|
833
1385
|
if (fragile)
|
|
@@ -836,6 +1388,39 @@ export class TaplineTools {
|
|
|
836
1388
|
}
|
|
837
1389
|
return response;
|
|
838
1390
|
}
|
|
1391
|
+
/**
|
|
1392
|
+
* When the agent's ref was saved as a testID, label or text that more than one element on screen has, add
|
|
1393
|
+
* the element's `index` among them (reading order), so the replay picks the same one instead of
|
|
1394
|
+
* answering `ambiguous`. One extra read, only while recording and only for ref targets.
|
|
1395
|
+
*/
|
|
1396
|
+
async withIndexIfShared(original, chosen, result, sessionId) {
|
|
1397
|
+
const before = typeof original === "object" && original !== null ? original.target : undefined;
|
|
1398
|
+
const target = chosen["target"];
|
|
1399
|
+
if (typeof before !== "object" || before === null || !("ref" in before))
|
|
1400
|
+
return chosen;
|
|
1401
|
+
if (typeof target !== "object" || target === null || "ref" in target || "index" in target)
|
|
1402
|
+
return chosen;
|
|
1403
|
+
const element = typeof result === "object" && result !== null && "element" in result
|
|
1404
|
+
? elementSchema.safeParse(result.element)
|
|
1405
|
+
: null;
|
|
1406
|
+
if (!element?.success)
|
|
1407
|
+
return chosen;
|
|
1408
|
+
const tree = treeResultSchema.safeParse(await this.host.request("tree", { peek: true }, { sessionId }).catch(() => null));
|
|
1409
|
+
if (!tree.success)
|
|
1410
|
+
return chosen;
|
|
1411
|
+
const t = target;
|
|
1412
|
+
const same = tree.data.elements.filter((el) => typeof t.testID === "string"
|
|
1413
|
+
? el.testID === t.testID
|
|
1414
|
+
: typeof t.label === "string"
|
|
1415
|
+
? labelOfElement(el).toLowerCase() === t.label.toLowerCase()
|
|
1416
|
+
: typeof t.text === "string"
|
|
1417
|
+
? el.text === t.text
|
|
1418
|
+
: false);
|
|
1419
|
+
if (same.length < 2)
|
|
1420
|
+
return chosen;
|
|
1421
|
+
const at = same.findIndex((el) => el.ref === element.data.ref);
|
|
1422
|
+
return at < 0 ? chosen : { ...chosen, target: { ...target, index: at } };
|
|
1423
|
+
}
|
|
839
1424
|
// -- the live viewer
|
|
840
1425
|
/** An element's frame (points) as 0..1 of the screen. */
|
|
841
1426
|
normalized(frame) {
|
|
@@ -859,15 +1444,43 @@ export class TaplineTools {
|
|
|
859
1444
|
...(point ? { point } : {}),
|
|
860
1445
|
};
|
|
861
1446
|
}
|
|
1447
|
+
/** One line in the action log (§12). Never fails the action. */
|
|
1448
|
+
logAction(by, tool, params, result, code, reason) {
|
|
1449
|
+
if (this.actionLog === null)
|
|
1450
|
+
return;
|
|
1451
|
+
const element = typeof result === "object" && result !== null && "element" in result
|
|
1452
|
+
? elementSchema.safeParse(result.element)
|
|
1453
|
+
: null;
|
|
1454
|
+
const session = this.host.current();
|
|
1455
|
+
this.actionLog.write({
|
|
1456
|
+
by,
|
|
1457
|
+
tool,
|
|
1458
|
+
...loggedTarget(params),
|
|
1459
|
+
// typed text never reaches the log, not even as the field's value (§12)
|
|
1460
|
+
...(element?.success ? { element: formatElement({ ...element.data, ref: "", value: null, placeholder: null }, { actions: false }) } : {}),
|
|
1461
|
+
ok: code === undefined,
|
|
1462
|
+
...(code !== undefined ? { code } : {}),
|
|
1463
|
+
...(reason !== undefined ? { reason } : {}),
|
|
1464
|
+
...(by === "agent" && this.actionStartedAt !== undefined ? { ms: Date.now() - this.actionStartedAt } : {}),
|
|
1465
|
+
...(session !== undefined ? { app: session.app.id ?? session.app.name } : {}),
|
|
1466
|
+
...(by === "agent" && this.batch !== undefined ? { batch: this.batch } : {}),
|
|
1467
|
+
...(by === "agent" ? (this.logExtra ?? {}) : {}),
|
|
1468
|
+
});
|
|
1469
|
+
if (by === "agent")
|
|
1470
|
+
this.logExtra = undefined;
|
|
1471
|
+
}
|
|
862
1472
|
showAction(by, method, result, params, code, reason) {
|
|
863
1473
|
const onAction = this.options.onAction;
|
|
864
|
-
if (onAction === undefined)
|
|
1474
|
+
if (onAction === undefined) {
|
|
1475
|
+
this.logAction(by, by === "agent" ? agentKind(method) : method, params, result, code, reason);
|
|
865
1476
|
return;
|
|
1477
|
+
}
|
|
866
1478
|
const target = typeof params === "object" && params !== null ? params.target : undefined;
|
|
867
1479
|
const t = typeof target === "object" && target !== null ? target : undefined;
|
|
868
1480
|
const screen = this.lastScreen;
|
|
869
1481
|
const point = t && typeof t.x === "number" && typeof t.y === "number" && screen ? { x: t.x / screen.w, y: t.y / screen.h } : undefined;
|
|
870
1482
|
const kind = by === "agent" ? agentKind(method) : method;
|
|
1483
|
+
this.logAction(by, kind, params, result, code, reason);
|
|
871
1484
|
const described = this.describeAction(kind, result, point);
|
|
872
1485
|
// A failed step has no element in its answer: name it from the target the agent gave, if it's words.
|
|
873
1486
|
const named = described.label ??
|
|
@@ -968,7 +1581,10 @@ export class TaplineTools {
|
|
|
968
1581
|
}
|
|
969
1582
|
};
|
|
970
1583
|
try {
|
|
971
|
-
|
|
1584
|
+
const outcome = this.options.control ? await this.options.control.runPerson(run) : await run();
|
|
1585
|
+
// The log (§12): a person's input too, its typed text only as a length.
|
|
1586
|
+
this.logAction("person", outcome.action?.kind ?? input.type, input.type === "text" ? { text: input.text } : {}, undefined, outcome.ok ? undefined : outcome.code, outcome.reason);
|
|
1587
|
+
return outcome;
|
|
972
1588
|
}
|
|
973
1589
|
catch (error) {
|
|
974
1590
|
return { ok: false, code: "internal", message: error instanceof Error ? error.message : String(error) };
|
|
@@ -1015,11 +1631,28 @@ export class TaplineTools {
|
|
|
1015
1631
|
const links = this.options.viewerLinks?.() ?? [];
|
|
1016
1632
|
if (links.length === 0)
|
|
1017
1633
|
return fail("Error unsupported: this host has no live viewer.");
|
|
1018
|
-
const
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1634
|
+
const open = this.options.openViewer;
|
|
1635
|
+
const local = links[0];
|
|
1636
|
+
let lines;
|
|
1637
|
+
if (open !== undefined && this.options.viewerLinkToAgent !== true && local !== undefined) {
|
|
1638
|
+
// The key in the link can approve a held-back touch: it goes to the browser, never to the agent.
|
|
1639
|
+
const opened = open(local);
|
|
1640
|
+
lines = [
|
|
1641
|
+
opened
|
|
1642
|
+
? "Opened the live view in a browser on this computer: the user watches the app and what you do there, and can tap and type too."
|
|
1643
|
+
: "Couldn't open a browser on this computer. Ask the user to start the MCP server with --viewer-link if they want the link.",
|
|
1644
|
+
"The link isn't shown to you: its key can approve touches that need a person's OK.",
|
|
1645
|
+
...(links.length > 1 ? ["For their own devices on Tailscale, the user finds that link in this server's log."] : []),
|
|
1646
|
+
"While the user acts in the live view, their touches go first and your actions wait.",
|
|
1647
|
+
];
|
|
1648
|
+
}
|
|
1649
|
+
else {
|
|
1650
|
+
lines = [
|
|
1651
|
+
"Live viewer (watch the app, see what you do, and let the user tap and type too):",
|
|
1652
|
+
...links.map((link) => `- ${link}`),
|
|
1653
|
+
"The link carries a key: give it to the user only. While they use the viewer, their touches go first and your actions wait.",
|
|
1654
|
+
];
|
|
1655
|
+
}
|
|
1023
1656
|
const feed = this.options.viewerFeed?.();
|
|
1024
1657
|
const session = this.host.current();
|
|
1025
1658
|
if (feed?.on === true && session !== undefined) {
|
|
@@ -1064,66 +1697,99 @@ export class TaplineTools {
|
|
|
1064
1697
|
else
|
|
1065
1698
|
content.push({ type: "image", data: image.data, mimeType: image.mime });
|
|
1066
1699
|
}
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
if (session === undefined)
|
|
1070
|
-
return this.notConnected();
|
|
1071
|
-
const name = args.name.trim();
|
|
1072
|
-
if (!SCRIPT_NAME.test(name))
|
|
1073
|
-
return fail("Error bad_params: name the reference with letters, digits, '.', '_' and '-'.");
|
|
1700
|
+
/** Where this app's reference pictures are kept, or an error answer. */
|
|
1701
|
+
referenceFolder(session) {
|
|
1074
1702
|
const dir = this.options.baselinesDir;
|
|
1075
1703
|
if (dir === undefined)
|
|
1076
1704
|
return fail("Error internal: no folder for reference pictures is configured.");
|
|
1077
|
-
|
|
1078
|
-
|
|
1705
|
+
return join(dir, (session.app.id ?? session.app.name).replace(/[^\w.-]/g, "_"));
|
|
1706
|
+
}
|
|
1707
|
+
/** The element list for naming regions and masking, read as a peek: the agent's refs and baseline must stay as they are. */
|
|
1708
|
+
async peekElements(sessionId) {
|
|
1709
|
+
try {
|
|
1710
|
+
const tree = treeResultSchema.safeParse(await this.host.request("tree", { peek: true }, { sessionId }));
|
|
1711
|
+
if (tree.success && tree.data.screen)
|
|
1712
|
+
return { scaleBase: tree.data.screen.width, elements: tree.data.elements };
|
|
1713
|
+
}
|
|
1714
|
+
catch {
|
|
1715
|
+
// positions stay in pixels
|
|
1716
|
+
}
|
|
1717
|
+
return { scaleBase: undefined, elements: [] };
|
|
1718
|
+
}
|
|
1719
|
+
/**
|
|
1720
|
+
* Takes the screen and compares it with the saved reference called `name`, never saving. `referencesDirs` are the
|
|
1721
|
+
* folders to look in, in order. A missing reference hands back the screenshot, so the caller may save it.
|
|
1722
|
+
*/
|
|
1723
|
+
async checkAgainstReference(session, name, options) {
|
|
1079
1724
|
let shot;
|
|
1080
1725
|
try {
|
|
1081
|
-
shot = await this.takeShot(session.id,
|
|
1726
|
+
shot = await this.takeShot(session.id, options.width, 0.9);
|
|
1082
1727
|
}
|
|
1083
1728
|
catch (error) {
|
|
1084
|
-
return
|
|
1729
|
+
return { kind: "error", line: describeError(error) };
|
|
1085
1730
|
}
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
}
|
|
1094
|
-
return text(`Saved the reference picture "${name}" (${shot.width ?? "?"}x${shot.height ?? "?"}) in ${path}. Next time, compare_screen { name: "${name}" } shows what changed.`);
|
|
1095
|
-
}
|
|
1096
|
-
let diff;
|
|
1731
|
+
this.lastMaskElements = undefined;
|
|
1732
|
+
if (!SCRIPT_NAME.test(name))
|
|
1733
|
+
return { kind: "error", line: BAD_REFERENCE_NAME };
|
|
1734
|
+
const path = options.referencesDirs.map((dir) => join(dir, `${name}.jpg`)).find((candidate) => existsSync(candidate));
|
|
1735
|
+
if (path === undefined)
|
|
1736
|
+
return { kind: "missing", shot };
|
|
1737
|
+
let reference;
|
|
1097
1738
|
try {
|
|
1098
|
-
|
|
1739
|
+
reference = { mime: "image/jpeg", data: readFileSync(path).toString("base64") };
|
|
1099
1740
|
}
|
|
1100
1741
|
catch (error) {
|
|
1101
|
-
|
|
1742
|
+
// A reference that can't be read (no permission, a folder in its place) is a failed check, never a crash.
|
|
1743
|
+
return { kind: "error", line: `Error bad_params: the reference "${name}" can't be read (${error instanceof Error ? error.message : String(error)}). To start over, use update: true.` };
|
|
1102
1744
|
}
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
// Name the regions in points, with the elements they touch, from a fresh element list.
|
|
1106
|
-
let scale = 1;
|
|
1107
|
-
let elements = [];
|
|
1745
|
+
let refSize;
|
|
1746
|
+
let shotSize;
|
|
1108
1747
|
try {
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1748
|
+
refSize = imageSize(reference);
|
|
1749
|
+
shotSize = imageSize(shot);
|
|
1750
|
+
}
|
|
1751
|
+
catch (error) {
|
|
1752
|
+
return { kind: "error", line: `Error bad_params: ${error instanceof Error ? error.message : String(error)} To start over, use update: true.` };
|
|
1753
|
+
}
|
|
1754
|
+
if (refSize.width !== shotSize.width || refSize.height !== shotSize.height) {
|
|
1755
|
+
return {
|
|
1756
|
+
kind: "size",
|
|
1757
|
+
message: `the reference "${name}" is ${refSize.width}x${refSize.height} px and this screen is ${shotSize.width}x${shotSize.height}; take it at the same width, or update: true.`,
|
|
1758
|
+
};
|
|
1759
|
+
}
|
|
1760
|
+
let seen;
|
|
1761
|
+
const notes = [];
|
|
1762
|
+
const mask = [];
|
|
1763
|
+
if (options.ignore.length > 0) {
|
|
1764
|
+
seen = await this.peekElements(session.id);
|
|
1765
|
+
this.lastMaskElements = seen.elements;
|
|
1766
|
+
if (seen.scaleBase === undefined) {
|
|
1767
|
+
return { kind: "error", line: "Error internal: couldn't read the screen's elements to paint out what ignore names." };
|
|
1114
1768
|
}
|
|
1769
|
+
const masked = ignoreRects(options.ignore, seen.elements, shotSize.width / seen.scaleBase, shotSize);
|
|
1770
|
+
mask.push(...masked.rects);
|
|
1771
|
+
notes.push(...masked.notes);
|
|
1115
1772
|
}
|
|
1116
|
-
|
|
1117
|
-
|
|
1773
|
+
let diff;
|
|
1774
|
+
try {
|
|
1775
|
+
diff = diffImages(reference, shot, { mask });
|
|
1776
|
+
}
|
|
1777
|
+
catch (error) {
|
|
1778
|
+
return { kind: "error", line: `Error bad_params: ${error instanceof Error ? error.message : String(error)} To start over, use update: true.` };
|
|
1118
1779
|
}
|
|
1780
|
+
// The pass bar compares the exact share; the rounded percent is only for showing (a few pixels round to 0%).
|
|
1781
|
+
const exactPercent = (diff.changedPixels / Math.max(1, diff.width * diff.height)) * 100;
|
|
1782
|
+
if (diff.changedPixels === 0 || exactPercent <= options.threshold) {
|
|
1783
|
+
return { kind: "match", percent: diff.changedPercent, notes };
|
|
1784
|
+
}
|
|
1785
|
+
// Name the regions in points, with the elements they touch.
|
|
1786
|
+
const found = seen ?? (await this.peekElements(session.id));
|
|
1787
|
+
const scale = found.scaleBase === undefined ? 1 : diff.width / found.scaleBase;
|
|
1119
1788
|
const unit = scale === 1 ? "pixels" : "points";
|
|
1120
|
-
const
|
|
1121
|
-
`${diff.changedPercent}% of the screen changed from the reference "${name}", in ${diff.boxes.length} region${diff.boxes.length === 1 ? "" : "s"} (${unit}):`,
|
|
1122
|
-
];
|
|
1123
|
-
for (const box of diff.boxes) {
|
|
1789
|
+
const regions = diff.boxes.map((box) => {
|
|
1124
1790
|
const b = { x: box.x / scale, y: box.y / scale, width: box.width / scale, height: box.height / scale };
|
|
1125
1791
|
// The smallest named elements there: a page-sized container overlaps every change.
|
|
1126
|
-
const near = elements
|
|
1792
|
+
const near = found.elements
|
|
1127
1793
|
.filter((e) => e.frame && e.frame.x < b.x + b.width && e.frame.x + e.frame.width > b.x && e.frame.y < b.y + b.height && e.frame.y + e.frame.height > b.y)
|
|
1128
1794
|
.filter((e) => (e.text ?? "") !== "" || (e.label ?? "") !== "" || (e.testID ?? "") !== "")
|
|
1129
1795
|
.map((e) => ({ e, area: (e.frame?.width ?? 0) * (e.frame?.height ?? 0) }))
|
|
@@ -1131,12 +1797,289 @@ export class TaplineTools {
|
|
|
1131
1797
|
.sort((x, y) => x.area - y.area)
|
|
1132
1798
|
.slice(0, 2)
|
|
1133
1799
|
.map(({ e }) => describeElement(e));
|
|
1134
|
-
|
|
1800
|
+
return { at: `@${Math.round(b.x)},${Math.round(b.y)} ${Math.round(b.width)}x${Math.round(b.height)}`, near };
|
|
1801
|
+
});
|
|
1802
|
+
const count = regions.length;
|
|
1803
|
+
const where = `in ${count} region${count === 1 ? "" : "s"} (${unit})`;
|
|
1804
|
+
const lines = [`Differs from the reference "${name}": ${diff.changedPercent}% changed, ${where}:`];
|
|
1805
|
+
for (const region of regions.slice(0, MAX_REGION_LINES)) {
|
|
1806
|
+
lines.push(`- ${region.at}${region.near.length > 0 ? `: ${region.near.join(", ")}` : ""}`);
|
|
1807
|
+
}
|
|
1808
|
+
if (count > MAX_REGION_LINES)
|
|
1809
|
+
lines.push(`(+${count - MAX_REGION_LINES} more regions)`);
|
|
1810
|
+
const listed = regions.slice(0, MAX_REGION_LINES).map((region) => region.at).join(", ");
|
|
1811
|
+
const reason = `${diff.changedPercent}% changed from the reference "${name}" (threshold ${options.threshold}%), ${where}: ${listed}${count > MAX_REGION_LINES ? `, +${count - MAX_REGION_LINES} more` : ""}.`;
|
|
1812
|
+
return { kind: "differs", percent: diff.changedPercent, lines, reason, image: diff.image, notes };
|
|
1813
|
+
}
|
|
1814
|
+
/** Why a check does not hold (for `run`, as a step or as `matches`), or null when it matches. */
|
|
1815
|
+
referenceReason(check, name) {
|
|
1816
|
+
switch (check.kind) {
|
|
1817
|
+
case "match":
|
|
1818
|
+
return null;
|
|
1819
|
+
case "missing":
|
|
1820
|
+
return { code: "expect_failed", message: `no reference "${name}" yet; save one with compare_screen { name: "${name}" }.` };
|
|
1821
|
+
case "size":
|
|
1822
|
+
return { code: "bad_params", message: check.message };
|
|
1823
|
+
case "error": {
|
|
1824
|
+
const { code, message } = parseError(check.line);
|
|
1825
|
+
return { code, message };
|
|
1826
|
+
}
|
|
1827
|
+
case "differs":
|
|
1828
|
+
return { code: "expect_failed", message: check.reason };
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
/** Whether the screen matches the reference `name` now (`expect.matches`): null when it does, else the reason. */
|
|
1832
|
+
async matchMissed(name, threshold, ignore) {
|
|
1833
|
+
this.lastMaskElements = undefined;
|
|
1834
|
+
if (!SCRIPT_NAME.test(name))
|
|
1835
|
+
return parseError(BAD_REFERENCE_NAME).message;
|
|
1836
|
+
const session = this.host.current();
|
|
1837
|
+
if (session === undefined)
|
|
1838
|
+
return "the app isn't connected";
|
|
1839
|
+
const folder = this.referenceFolder(session);
|
|
1840
|
+
if (typeof folder !== "string")
|
|
1841
|
+
return parseError(folder.content.map((block) => (block.type === "text" ? block.text : "")).join("\n")).message;
|
|
1842
|
+
if (this.replayContext?.update === true) {
|
|
1843
|
+
// --update-references: the check saves the screen into the script's folder, and holds.
|
|
1844
|
+
const saved = await this.saveReferenceHere(session, name, DEFAULT_SCREENSHOT_WIDTH, this.replayContext.referencesDir);
|
|
1845
|
+
return saved.isError === true ? parseError(saved.content.map((block) => (block.type === "text" ? block.text : "")).join("\n")).message : null;
|
|
1846
|
+
}
|
|
1847
|
+
const check = await this.checkAgainstReference(session, name, {
|
|
1848
|
+
threshold,
|
|
1849
|
+
ignore,
|
|
1850
|
+
width: DEFAULT_SCREENSHOT_WIDTH,
|
|
1851
|
+
referencesDirs: this.referencesDirs(folder),
|
|
1852
|
+
});
|
|
1853
|
+
this.keepDiff(check, name);
|
|
1854
|
+
return this.referenceReason(check, name)?.message ?? null;
|
|
1855
|
+
}
|
|
1856
|
+
/** Where a check looks for `<name>.jpg`: the folder next to a replayed script first, then the app's own store. */
|
|
1857
|
+
referencesDirs(folder) {
|
|
1858
|
+
return this.replayContext !== null ? [this.replayContext.referencesDir, folder] : [folder];
|
|
1859
|
+
}
|
|
1860
|
+
/** In a replay, a failed check leaves its red picture at `<script>.diff/<name>.jpg`. */
|
|
1861
|
+
keepDiff(check, name) {
|
|
1862
|
+
if (this.replayContext === null || check.kind !== "differs" || !SCRIPT_NAME.test(name))
|
|
1863
|
+
return;
|
|
1864
|
+
const path = join(this.replayContext.diffDir, `${name}.jpg`);
|
|
1865
|
+
try {
|
|
1866
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1867
|
+
writeFileSync(path, Buffer.from(check.image.data, "base64"));
|
|
1868
|
+
this.diffSaved = path;
|
|
1869
|
+
}
|
|
1870
|
+
catch {
|
|
1871
|
+
// the failure still reads without the picture
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
/** Takes the screen and saves it as the reference `name` in `dir`. */
|
|
1875
|
+
async saveReferenceHere(session, name, width, dir) {
|
|
1876
|
+
try {
|
|
1877
|
+
return this.writeReference(dir, name, await this.takeShot(session.id, width, 0.9));
|
|
1878
|
+
}
|
|
1879
|
+
catch (error) {
|
|
1880
|
+
return fail(describeError(error));
|
|
1881
|
+
}
|
|
1882
|
+
}
|
|
1883
|
+
writeReference(folder, name, shot) {
|
|
1884
|
+
// Every write checks the name itself, whatever the caller checked: no way out of the references folder.
|
|
1885
|
+
if (!SCRIPT_NAME.test(name))
|
|
1886
|
+
return fail(BAD_REFERENCE_NAME);
|
|
1887
|
+
const path = join(folder, `${name}.jpg`);
|
|
1888
|
+
try {
|
|
1889
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
1890
|
+
writeFileSync(path, Buffer.from(shot.data, "base64"), { mode: 0o600 });
|
|
1891
|
+
}
|
|
1892
|
+
catch (error) {
|
|
1893
|
+
return fail(`Error internal: couldn't save the reference: ${error instanceof Error ? error.message : String(error)}`);
|
|
1894
|
+
}
|
|
1895
|
+
let size = `${shot.width ?? "?"}x${shot.height ?? "?"}`;
|
|
1896
|
+
try {
|
|
1897
|
+
const real = imageSize(shot);
|
|
1898
|
+
size = `${real.width}x${real.height}`;
|
|
1899
|
+
}
|
|
1900
|
+
catch {
|
|
1901
|
+
// the app's own size stays
|
|
1902
|
+
}
|
|
1903
|
+
return text(`Saved the reference "${name}" (${size} px).`);
|
|
1904
|
+
}
|
|
1905
|
+
/** A `compare_screen` step of `run`: it never saves; it holds on a match and stops the batch otherwise. */
|
|
1906
|
+
async compareStep(args) {
|
|
1907
|
+
this.lastMaskElements = undefined;
|
|
1908
|
+
const session = this.host.current();
|
|
1909
|
+
if (session === undefined)
|
|
1910
|
+
return this.notConnected();
|
|
1911
|
+
const name = args.name.trim();
|
|
1912
|
+
if (!SCRIPT_NAME.test(name))
|
|
1913
|
+
return fail("Error bad_params: name the reference with letters, digits, '.', '_' and '-'.");
|
|
1914
|
+
const folder = this.referenceFolder(session);
|
|
1915
|
+
if (typeof folder !== "string")
|
|
1916
|
+
return folder;
|
|
1917
|
+
if (this.replayContext?.update === true) {
|
|
1918
|
+
return this.saveReferenceHere(session, name, args.width ?? DEFAULT_SCREENSHOT_WIDTH, this.replayContext.referencesDir);
|
|
1919
|
+
}
|
|
1920
|
+
const check = await this.checkAgainstReference(session, name, {
|
|
1921
|
+
threshold: args.threshold ?? DEFAULT_MATCH_THRESHOLD,
|
|
1922
|
+
ignore: args.ignore ?? [],
|
|
1923
|
+
width: args.width ?? DEFAULT_SCREENSHOT_WIDTH,
|
|
1924
|
+
referencesDirs: this.referencesDirs(folder),
|
|
1925
|
+
});
|
|
1926
|
+
this.keepDiff(check, name);
|
|
1927
|
+
const why = this.referenceReason(check, name);
|
|
1928
|
+
if (why !== null)
|
|
1929
|
+
return fail(`Error ${why.code}: ${why.message}`);
|
|
1930
|
+
if (check.kind !== "match")
|
|
1931
|
+
return fail("Error internal: the comparison gave no answer.");
|
|
1932
|
+
// A check that held inside `run` is kept in a recording too, with its options.
|
|
1933
|
+
const recorded = await this.recordCompare({
|
|
1934
|
+
name,
|
|
1935
|
+
...(args.threshold !== undefined ? { threshold: args.threshold } : {}),
|
|
1936
|
+
...(args.ignore !== undefined ? { ignore: args.ignore } : {}),
|
|
1937
|
+
...(args.width !== undefined ? { width: args.width } : {}),
|
|
1938
|
+
});
|
|
1939
|
+
if (recorded !== null)
|
|
1940
|
+
return fail(recorded);
|
|
1941
|
+
return text([`Matches the reference "${name}": ${check.percent}% changed.`, ...check.notes.map((n) => `Note: ${n}`)].join("\n"));
|
|
1942
|
+
}
|
|
1943
|
+
/**
|
|
1944
|
+
* Saves a visual check that held as a `compareScreen` step of the recording, when one is running: the name, and the
|
|
1945
|
+
* threshold, the ignore list (refs made stable) and the width when given. `update` is never written. Answers an
|
|
1946
|
+
* error line when an ignore ref can't be kept in a script, else null (also when nothing is recording).
|
|
1947
|
+
*/
|
|
1948
|
+
async recordCompare(check) {
|
|
1949
|
+
if (this.recording === null || this.replaying)
|
|
1950
|
+
return null;
|
|
1951
|
+
const session = this.host.current();
|
|
1952
|
+
let kept = check.ignore;
|
|
1953
|
+
if (session !== undefined && kept !== undefined && kept.length > 0) {
|
|
1954
|
+
const stable = await this.stableIgnore(session.id, kept, this.lastMaskElements);
|
|
1955
|
+
if (typeof stable === "string")
|
|
1956
|
+
return stable;
|
|
1957
|
+
kept = stable;
|
|
1958
|
+
}
|
|
1959
|
+
if (this.recording === null)
|
|
1960
|
+
return null;
|
|
1961
|
+
this.recording.steps.push({
|
|
1962
|
+
method: "compareScreen",
|
|
1963
|
+
params: {
|
|
1964
|
+
name: check.name.trim(),
|
|
1965
|
+
...(check.threshold !== undefined ? { threshold: check.threshold } : {}),
|
|
1966
|
+
...(kept !== undefined && kept.length > 0 ? { ignore: kept } : {}),
|
|
1967
|
+
...(check.width !== undefined ? { width: check.width } : {}),
|
|
1968
|
+
},
|
|
1969
|
+
});
|
|
1970
|
+
return null;
|
|
1971
|
+
}
|
|
1972
|
+
/**
|
|
1973
|
+
* `ignore` as a script keeps it (AGENT-TOOLS 1.9 §14.2): a ref becomes the element's id, else label, else text,
|
|
1974
|
+
* read from the list now. A string is the error answer when a ref can't be kept.
|
|
1975
|
+
*/
|
|
1976
|
+
async stableIgnore(sessionId, ignore, known) {
|
|
1977
|
+
if (!ignore.some((item) => !("width" in item) && item.ref !== undefined))
|
|
1978
|
+
return ignore;
|
|
1979
|
+
// The elements the comparison painted out when it had them, so the script keeps exactly that mask.
|
|
1980
|
+
const seen = known !== undefined ? { elements: known } : await this.peekElements(sessionId);
|
|
1981
|
+
const kept = [];
|
|
1982
|
+
for (const item of ignore) {
|
|
1983
|
+
if ("width" in item || item.ref === undefined) {
|
|
1984
|
+
kept.push(item);
|
|
1985
|
+
continue;
|
|
1986
|
+
}
|
|
1987
|
+
const element = seen.elements.find((e) => e.ref === item.ref);
|
|
1988
|
+
if (element === undefined)
|
|
1989
|
+
return `Error bad_params: ignore ${item.ref} isn't on the screen, so it can't be saved in a script.`;
|
|
1990
|
+
// When several elements share the name, the saved item keeps which one it was (`index`, counted as ignoreRects
|
|
1991
|
+
// counts: elements with a frame, in list order), so a replay paints out that one and not all of them.
|
|
1992
|
+
const framed = seen.elements.filter((e) => e.frame !== undefined);
|
|
1993
|
+
const withIndex = (saved, same) => {
|
|
1994
|
+
const matching = framed.filter(same);
|
|
1995
|
+
const at = matching.indexOf(element);
|
|
1996
|
+
return matching.length > 1 && at >= 0 ? { ...saved, index: at } : saved;
|
|
1997
|
+
};
|
|
1998
|
+
if (typeof element.testID === "string" && element.testID !== "") {
|
|
1999
|
+
const id = element.testID;
|
|
2000
|
+
kept.push(withIndex({ id }, (e) => e.testID === id));
|
|
2001
|
+
}
|
|
2002
|
+
else if (typeof element.label === "string" && element.label !== "") {
|
|
2003
|
+
const label = element.label.toLowerCase();
|
|
2004
|
+
kept.push(withIndex({ label: element.label }, (e) => labelOfElement(e).toLowerCase() === label));
|
|
2005
|
+
}
|
|
2006
|
+
else if (typeof element.text === "string" && element.text !== "") {
|
|
2007
|
+
const text = element.text.toLowerCase();
|
|
2008
|
+
kept.push(withIndex({ text: element.text }, (e) => (e.text ?? "").toLowerCase() === text));
|
|
2009
|
+
}
|
|
2010
|
+
else {
|
|
2011
|
+
return `Error bad_params: ignore ${describeElement(element)} has no id, label or text, so a script couldn't find it again; give it a testID, or ignore a rect {x, y, width, height}.`;
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
2014
|
+
return kept;
|
|
2015
|
+
}
|
|
2016
|
+
async compareScreen(args) {
|
|
2017
|
+
// Each check starts with no mask list: an update, or a first save, never records from an earlier comparison.
|
|
2018
|
+
this.lastMaskElements = undefined;
|
|
2019
|
+
const recording = this.recording !== null && !this.replaying;
|
|
2020
|
+
const session = this.host.current();
|
|
2021
|
+
// An ignore ref that a script couldn't find again is refused before anything is compared.
|
|
2022
|
+
if (recording && session !== undefined && args.ignore !== undefined && args.ignore.length > 0) {
|
|
2023
|
+
const stable = await this.stableIgnore(session.id, args.ignore);
|
|
2024
|
+
if (typeof stable === "string")
|
|
2025
|
+
return fail(stable);
|
|
2026
|
+
}
|
|
2027
|
+
const answer = await this.compareScreenNow(args);
|
|
2028
|
+
if (recording && answer.isError !== true) {
|
|
2029
|
+
// `update` is never written: the script checks against the reference it carries.
|
|
2030
|
+
const recorded = await this.recordCompare({
|
|
2031
|
+
name: args.name,
|
|
2032
|
+
...(args.threshold !== undefined ? { threshold: args.threshold } : {}),
|
|
2033
|
+
...(args.ignore !== undefined ? { ignore: args.ignore } : {}),
|
|
2034
|
+
...(args.width !== undefined ? { width: args.width } : {}),
|
|
2035
|
+
});
|
|
2036
|
+
if (recorded !== null)
|
|
2037
|
+
return fail(recorded);
|
|
2038
|
+
}
|
|
2039
|
+
return answer;
|
|
2040
|
+
}
|
|
2041
|
+
async compareScreenNow(args) {
|
|
2042
|
+
const session = this.host.current();
|
|
2043
|
+
if (session === undefined)
|
|
2044
|
+
return this.notConnected();
|
|
2045
|
+
const name = args.name.trim();
|
|
2046
|
+
if (!SCRIPT_NAME.test(name))
|
|
2047
|
+
return fail("Error bad_params: name the reference with letters, digits, '.', '_' and '-'.");
|
|
2048
|
+
const folder = this.referenceFolder(session);
|
|
2049
|
+
if (typeof folder !== "string")
|
|
2050
|
+
return folder;
|
|
2051
|
+
const width = args.width ?? DEFAULT_SCREENSHOT_WIDTH;
|
|
2052
|
+
const save = (shot) => this.writeReference(folder, name, shot);
|
|
2053
|
+
if (args.update === true) {
|
|
2054
|
+
try {
|
|
2055
|
+
return save(await this.takeShot(session.id, width, 0.9));
|
|
2056
|
+
}
|
|
2057
|
+
catch (error) {
|
|
2058
|
+
return fail(describeError(error));
|
|
2059
|
+
}
|
|
2060
|
+
}
|
|
2061
|
+
const check = await this.checkAgainstReference(session, name, {
|
|
2062
|
+
threshold: args.threshold ?? DEFAULT_MATCH_THRESHOLD,
|
|
2063
|
+
ignore: args.ignore ?? [],
|
|
2064
|
+
width,
|
|
2065
|
+
referencesDirs: [folder],
|
|
2066
|
+
});
|
|
2067
|
+
switch (check.kind) {
|
|
2068
|
+
case "missing":
|
|
2069
|
+
return save(check.shot);
|
|
2070
|
+
case "error":
|
|
2071
|
+
return fail(check.line);
|
|
2072
|
+
case "size":
|
|
2073
|
+
return fail(`Error bad_params: ${check.message}`);
|
|
2074
|
+
case "match":
|
|
2075
|
+
return text([`Matches the reference "${name}": ${check.percent}% changed.`, ...check.notes.map((n) => `Note: ${n}`)].join("\n"));
|
|
2076
|
+
case "differs": {
|
|
2077
|
+
const lines = [...check.lines, "The image shows the screen now, faded, with the changed pixels in red.", ...check.notes.map((n) => `Note: ${n}`)];
|
|
2078
|
+
const content = [];
|
|
2079
|
+
this.deliver(session.id, check.image, lines, content);
|
|
2080
|
+
return { content: [{ type: "text", text: lines.join("\n") }, ...content] };
|
|
2081
|
+
}
|
|
1135
2082
|
}
|
|
1136
|
-
lines.push("The image shows the screen now, faded, with the changed pixels in red.");
|
|
1137
|
-
const content = [];
|
|
1138
|
-
this.deliver(session.id, diff.image, lines, content);
|
|
1139
|
-
return { content: [{ type: "text", text: lines.join("\n") }, ...content] };
|
|
1140
2083
|
}
|
|
1141
2084
|
async filmstrip(args) {
|
|
1142
2085
|
const session = this.host.current();
|
|
@@ -1151,8 +2094,13 @@ export class TaplineTools {
|
|
|
1151
2094
|
const started = Date.now();
|
|
1152
2095
|
if (args.tap !== undefined) {
|
|
1153
2096
|
// A screenshot takes a few hundred ms on a phone, so frame 1 is the "before".
|
|
2097
|
+
// The tap is a touch like any other: the safety policy decides first.
|
|
2098
|
+
const pressParams = { target: toWireTarget(args.tap) };
|
|
2099
|
+
const gate = await this.policyGate(session, "tap", "press", pressParams, listsFor(this.policy, session.app));
|
|
2100
|
+
if ("stop" in gate)
|
|
2101
|
+
return gate.stop;
|
|
1154
2102
|
frames.push(await this.takeShot(session.id, frameWidth, 0.8));
|
|
1155
|
-
await this.host.request("press",
|
|
2103
|
+
await this.host.request("press", pressParams, { sessionId: session.id });
|
|
1156
2104
|
}
|
|
1157
2105
|
for (let i = frames.length; i < count; i++) {
|
|
1158
2106
|
const at = Date.now();
|
|
@@ -1178,9 +2126,10 @@ export class TaplineTools {
|
|
|
1178
2126
|
return dir === undefined || !SCRIPT_NAME.test(name) ? null : join(dir, `${name}.json`);
|
|
1179
2127
|
}
|
|
1180
2128
|
async record(args) {
|
|
1181
|
-
|
|
2129
|
+
const op = args.action ?? (args.start === true ? "start" : args.stop === true ? "stop" : "status");
|
|
2130
|
+
switch (op) {
|
|
1182
2131
|
case "start": {
|
|
1183
|
-
const name = (args.name ?? "").trim()
|
|
2132
|
+
const name = (args.name ?? "").trim() || `session-${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, "")}`;
|
|
1184
2133
|
if (this.scriptPath(name) === null) {
|
|
1185
2134
|
return fail("Error bad_params: give the script a name of letters, digits, '.', '_' and '-' (e.g. \"checkout\").");
|
|
1186
2135
|
}
|
|
@@ -1190,7 +2139,7 @@ export class TaplineTools {
|
|
|
1190
2139
|
if (session === undefined)
|
|
1191
2140
|
return this.notConnected();
|
|
1192
2141
|
this.recording = { name, app: { name: session.app.name, ...(session.app.id ? { id: session.app.id } : {}) }, steps: [] };
|
|
1193
|
-
return text(`Recording "${name}" on ${session.app.name}. Every action from now on is a step; wait_for steps check the app is where it should be. Prefer testIDs: they replay best. Stop with record {
|
|
2142
|
+
return text(`Recording "${name}" on ${session.app.name}. Every action from now on is a step; wait_for steps check the app is where it should be. Prefer testIDs: they replay best. Stop with record { stop: true }.`);
|
|
1194
2143
|
}
|
|
1195
2144
|
case "status":
|
|
1196
2145
|
return text(this.recording === null
|
|
@@ -1204,16 +2153,17 @@ export class TaplineTools {
|
|
|
1204
2153
|
case "stop": {
|
|
1205
2154
|
const recording = this.recording;
|
|
1206
2155
|
if (recording === null)
|
|
1207
|
-
return fail("Error bad_params: not recording; start with record {
|
|
1208
|
-
const path = this.scriptPath(recording.name);
|
|
2156
|
+
return fail("Error bad_params: not recording; start with record { start: true, name }.");
|
|
2157
|
+
const path = args.path !== undefined && args.path.trim() !== "" ? args.path.trim() : this.scriptPath(recording.name);
|
|
1209
2158
|
if (path === null)
|
|
1210
|
-
return fail("Error internal: no scripts folder is configured.");
|
|
2159
|
+
return fail("Error internal: no scripts folder is configured; give a path.");
|
|
1211
2160
|
const script = {
|
|
2161
|
+
format: "agent-tools-script",
|
|
1212
2162
|
version: 1,
|
|
1213
2163
|
name: recording.name,
|
|
1214
|
-
app: recording.app,
|
|
1215
|
-
|
|
1216
|
-
steps: recording.steps,
|
|
2164
|
+
app: recording.app.id ?? recording.app.name,
|
|
2165
|
+
fresh: true,
|
|
2166
|
+
steps: scriptSteps(recording.steps),
|
|
1217
2167
|
};
|
|
1218
2168
|
try {
|
|
1219
2169
|
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
@@ -1224,20 +2174,60 @@ export class TaplineTools {
|
|
|
1224
2174
|
}
|
|
1225
2175
|
this.recording = null;
|
|
1226
2176
|
const count = script.steps.length;
|
|
1227
|
-
|
|
1228
|
-
script.steps.
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
2177
|
+
// AGENT-TOOLS 1.9 §14.3: the references the steps use travel with the script.
|
|
2178
|
+
const copied = this.copyReferences(script.steps, recording.app, path);
|
|
2179
|
+
const withPictures = copied === null ? "" : `, and ${copied.count} reference picture${copied.count === 1 ? "" : "s"} to ${copied.folder}`;
|
|
2180
|
+
const lines = [`Saved ${count} step${count === 1 ? "" : "s"} to ${path}${withPictures}.`];
|
|
2181
|
+
const ref = refStep(script.steps);
|
|
2182
|
+
if (ref !== null) {
|
|
2183
|
+
lines.push(`Note: step ${ref + 1} targets a ref, which won't exist on a new run, so replay refuses the script. Give that element a testID, or record it again by its text or label.`);
|
|
2184
|
+
}
|
|
2185
|
+
const secrets = recording.steps.filter((step) => step.secret === true).length;
|
|
2186
|
+
if (secrets > 0) {
|
|
2187
|
+
lines.push(`${secrets} step${secrets === 1 ? " types" : "s type"} into a password field: that text isn't saved. Put a test password into the script before you replay it.`);
|
|
2188
|
+
}
|
|
2189
|
+
if (recording.steps.some((step) => step.method === "type" && step.secret !== true)) {
|
|
2190
|
+
lines.push("The script keeps the other text typed in it: keep it private.");
|
|
1236
2191
|
}
|
|
1237
2192
|
return text(lines.join("\n"));
|
|
1238
2193
|
}
|
|
1239
2194
|
}
|
|
1240
2195
|
}
|
|
2196
|
+
/** `<script dir>/<script name>.references` and `.diff` for a script file. */
|
|
2197
|
+
scriptSideFolder(path, kind) {
|
|
2198
|
+
return join(dirname(path), `${basename(path).replace(/\.json$/i, "")}.${kind}`);
|
|
2199
|
+
}
|
|
2200
|
+
/** Copies each reference the steps use from the app's store next to the script; null when there are none. */
|
|
2201
|
+
copyReferences(steps, app, scriptPath) {
|
|
2202
|
+
const dir = this.options.baselinesDir;
|
|
2203
|
+
if (dir === undefined)
|
|
2204
|
+
return null;
|
|
2205
|
+
const names = new Set();
|
|
2206
|
+
for (const step of steps) {
|
|
2207
|
+
const p = step;
|
|
2208
|
+
if (step.tool === "compare_screen" && typeof p["name"] === "string" && SCRIPT_NAME.test(p["name"]))
|
|
2209
|
+
names.add(p["name"]);
|
|
2210
|
+
if (step.expect?.matches !== undefined && SCRIPT_NAME.test(step.expect.matches))
|
|
2211
|
+
names.add(step.expect.matches);
|
|
2212
|
+
}
|
|
2213
|
+
const store = join(dir, (app.id ?? app.name).replace(/[^\w.-]/g, "_"));
|
|
2214
|
+
const folder = this.scriptSideFolder(scriptPath, "references");
|
|
2215
|
+
let count = 0;
|
|
2216
|
+
for (const name of names) {
|
|
2217
|
+
const from = join(store, `${name}.jpg`);
|
|
2218
|
+
if (!existsSync(from))
|
|
2219
|
+
continue;
|
|
2220
|
+
try {
|
|
2221
|
+
mkdirSync(folder, { recursive: true });
|
|
2222
|
+
copyFileSync(from, join(folder, `${name}.jpg`));
|
|
2223
|
+
count++;
|
|
2224
|
+
}
|
|
2225
|
+
catch {
|
|
2226
|
+
// the script still saved; the picture stays in the store
|
|
2227
|
+
}
|
|
2228
|
+
}
|
|
2229
|
+
return count === 0 ? null : { count, folder };
|
|
2230
|
+
}
|
|
1241
2231
|
scriptList() {
|
|
1242
2232
|
const dir = this.options.scriptsDir;
|
|
1243
2233
|
if (dir === undefined || !existsSync(dir))
|
|
@@ -1245,11 +2235,10 @@ export class TaplineTools {
|
|
|
1245
2235
|
const lines = [];
|
|
1246
2236
|
for (const file of readdirSync(dir).filter((f) => f.endsWith(".json")).sort()) {
|
|
1247
2237
|
try {
|
|
1248
|
-
const
|
|
1249
|
-
if (
|
|
2238
|
+
const s = readScript(readFileSync(join(dir, file), "utf8"));
|
|
2239
|
+
if (s === null)
|
|
1250
2240
|
continue;
|
|
1251
|
-
|
|
1252
|
-
lines.push(`- ${s.name} (${s.steps.length} step${s.steps.length === 1 ? "" : "s"}, recorded ${s.recordedAt.slice(0, 10)} on ${s.app.name})`);
|
|
2241
|
+
lines.push(`- ${s.name} (${s.steps.length} step${s.steps.length === 1 ? "" : "s"}${s.app ? `, ${s.app}` : ""})`);
|
|
1253
2242
|
}
|
|
1254
2243
|
catch {
|
|
1255
2244
|
// skip unreadable files
|
|
@@ -1257,66 +2246,68 @@ export class TaplineTools {
|
|
|
1257
2246
|
}
|
|
1258
2247
|
return lines.length === 0 ? "No saved scripts." : ["Saved scripts:", ...lines].join("\n");
|
|
1259
2248
|
}
|
|
1260
|
-
async replay(args) {
|
|
2249
|
+
async replay(args, options = {}) {
|
|
1261
2250
|
const name = (args.name ?? "").trim();
|
|
1262
|
-
|
|
2251
|
+
const given = (args.path ?? "").trim();
|
|
2252
|
+
if (name === "" && given === "")
|
|
1263
2253
|
return text(this.scriptList());
|
|
1264
2254
|
if (this.recording !== null)
|
|
1265
2255
|
return fail(`Error bad_params: stop recording "${this.recording.name}" before replaying.`);
|
|
1266
|
-
const path = this.scriptPath(name);
|
|
2256
|
+
const path = given !== "" ? given : this.scriptPath(name);
|
|
1267
2257
|
if (path === null || !existsSync(path))
|
|
1268
|
-
return fail(`Error not_found: no script "${name}".\n${this.scriptList()}`);
|
|
2258
|
+
return fail(`Error not_found: no script "${given || name}".\n${this.scriptList()}`);
|
|
1269
2259
|
let script;
|
|
1270
2260
|
try {
|
|
1271
|
-
|
|
1272
|
-
if (!parsed.success)
|
|
1273
|
-
return fail(`Error bad_params: ${path} isn't a Tapline script.`);
|
|
1274
|
-
script = parsed.data;
|
|
2261
|
+
script = readScript(readFileSync(path, "utf8"));
|
|
1275
2262
|
}
|
|
1276
2263
|
catch (error) {
|
|
1277
2264
|
return fail(`Error internal: couldn't read ${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1278
2265
|
}
|
|
2266
|
+
if (script === null)
|
|
2267
|
+
return fail(`Error bad_params: ${path} isn't a script (format agent-tools-script, version 1).`);
|
|
2268
|
+
const ref = refStep(script.steps);
|
|
2269
|
+
if (ref !== null)
|
|
2270
|
+
return fail(`Error bad_params: step ${ref + 1} targets a ref; a script needs an id, label, text or point.`);
|
|
2271
|
+
if (script.steps.length === 0)
|
|
2272
|
+
return text(`Ran 0 of 0 steps in 0.0 s.`);
|
|
1279
2273
|
const session = this.host.current();
|
|
1280
2274
|
if (session === undefined)
|
|
1281
2275
|
return this.notConnected();
|
|
1282
|
-
const
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
2276
|
+
const notes = [];
|
|
2277
|
+
script.steps.forEach((step, i) => {
|
|
2278
|
+
const target = step["target"];
|
|
2279
|
+
if (typeof target === "object" && target !== null && "x" in target) {
|
|
2280
|
+
notes.push(`Note (step ${i + 1}): a point target; it breaks when the layout changes.`);
|
|
2281
|
+
}
|
|
2282
|
+
});
|
|
2283
|
+
if (script.fresh !== false) {
|
|
2284
|
+
// Tapline can't close and open the app: it starts its JavaScript again (a development build).
|
|
2285
|
+
const fresh = await this.reload({});
|
|
2286
|
+
const said = fresh.content.map((block) => (block.type === "text" ? block.text : "")).join("\n");
|
|
2287
|
+
if (fresh.isError === true)
|
|
2288
|
+
notes.push("Note: couldn't start the app fresh (a release build has no reload); the script ran on the screen as it was.");
|
|
2289
|
+
else if (said.startsWith("Reload requested, but"))
|
|
2290
|
+
notes.push("Note: the app hadn't reconnected after starting fresh when the steps began.");
|
|
2291
|
+
}
|
|
1286
2292
|
this.replaying = true;
|
|
2293
|
+
this.replayContext = {
|
|
2294
|
+
referencesDir: this.scriptSideFolder(path, "references"),
|
|
2295
|
+
diffDir: this.scriptSideFolder(path, "diff"),
|
|
2296
|
+
update: options.updateReferences === true,
|
|
2297
|
+
};
|
|
1287
2298
|
try {
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
continue;
|
|
1296
|
-
}
|
|
1297
|
-
const result = await this.act(step.method, step.params, () => "", true);
|
|
1298
|
-
const problem = result.isError
|
|
1299
|
-
? result.content.map((block) => (block.type === "text" ? block.text : "")).join("\n").trim()
|
|
1300
|
-
: step.route !== undefined &&
|
|
1301
|
-
this.lastLookRoute !== undefined &&
|
|
1302
|
-
step.route.join(" › ") !== this.lastLookRoute.join(" › ")
|
|
1303
|
-
? `It ended on ${this.lastLookRoute.join(" › ")}; the recording ended on ${step.route.join(" › ")}.`
|
|
1304
|
-
: null;
|
|
1305
|
-
if (problem !== null) {
|
|
1306
|
-
return fail([
|
|
1307
|
-
`Replay "${script.name}" failed at step ${i + 1} of ${total}: ${describeStep(step)}`,
|
|
1308
|
-
problem,
|
|
1309
|
-
...(lines.length > 0 ? ["Steps before it passed:", ...lines] : []),
|
|
1310
|
-
...(step.fragile ? ["Note: this step targets a ref from the recording run; give the element a testID."] : []),
|
|
1311
|
-
].join("\n"));
|
|
1312
|
-
}
|
|
1313
|
-
lines.push(`${label.replace(/^(\d+) /, "$1 ✓ ")} (${seconds(Date.now() - at)} s)`);
|
|
1314
|
-
}
|
|
2299
|
+
const answer = await this.run({ steps: script.steps, look: args.look, screenshot: args.screenshot });
|
|
2300
|
+
if (notes.length === 0)
|
|
2301
|
+
return answer;
|
|
2302
|
+
const [top, ...rest] = answer.content;
|
|
2303
|
+
return top !== undefined && top.type === "text"
|
|
2304
|
+
? { ...answer, content: [{ type: "text", text: [top.text.trimEnd(), ...notes].join("\n") }, ...rest] }
|
|
2305
|
+
: answer;
|
|
1315
2306
|
}
|
|
1316
2307
|
finally {
|
|
1317
2308
|
this.replaying = false;
|
|
2309
|
+
this.replayContext = null;
|
|
1318
2310
|
}
|
|
1319
|
-
return text([`Replayed "${script.name}" on ${session.app.name}: ${total} of ${total} steps passed in ${seconds(Date.now() - started)} s.`, ...lines].join("\n"));
|
|
1320
2311
|
}
|
|
1321
2312
|
async withLook(session, line, screenshot = false) {
|
|
1322
2313
|
// The session may have been replaced (app restarted) in the meantime.
|
|
@@ -1329,21 +2320,275 @@ export class TaplineTools {
|
|
|
1329
2320
|
const head = first !== undefined && first.type === "text" ? first.text : "";
|
|
1330
2321
|
return { content: [{ type: "text", text: `${line}\n${head}` }, ...rest] };
|
|
1331
2322
|
}
|
|
1332
|
-
/**
|
|
1333
|
-
|
|
2323
|
+
/**
|
|
2324
|
+
* The element a result carries under `key`, for an answer's first line: without its actions, position or
|
|
2325
|
+
* size (AGENT-TOOLS 1.8 §13 rule 1; the change block or list has them). `withoutValue` for type_text,
|
|
2326
|
+
* whose sentence says the value (rule 5).
|
|
2327
|
+
*/
|
|
2328
|
+
elementName(result, key = "element", withoutValue = false) {
|
|
1334
2329
|
if (typeof result === "object" && result !== null && key in result) {
|
|
1335
2330
|
const element = elementSchema.safeParse(result[key]);
|
|
1336
|
-
if (element.success)
|
|
1337
|
-
|
|
2331
|
+
if (element.success) {
|
|
2332
|
+
const shown = withoutValue ? { ...element.data, value: null, placeholder: null } : element.data;
|
|
2333
|
+
return formatElement(shown, { actions: false, frame: false });
|
|
2334
|
+
}
|
|
1338
2335
|
}
|
|
1339
2336
|
return "element";
|
|
1340
2337
|
}
|
|
2338
|
+
// -- the safety policy (AGENT-TOOLS 1.8 §11)
|
|
2339
|
+
/**
|
|
2340
|
+
* A guarded touch: the app resolves the target in a dry run, the policy's words are matched against the
|
|
2341
|
+
* element's own label and text, and a confirm word waits for a person's OK in the live view. The check runs
|
|
2342
|
+
* inside the agent's turn, right before the touch, so nothing a person does in between can change what is
|
|
2343
|
+
* touched; when the check can't be made, nothing is touched (fail closed).
|
|
2344
|
+
*/
|
|
2345
|
+
async guarded(tool, method, params, dryRun, perform) {
|
|
2346
|
+
const session = this.host.current();
|
|
2347
|
+
if (session === undefined)
|
|
2348
|
+
return this.notConnected();
|
|
2349
|
+
this.logExtra = undefined;
|
|
2350
|
+
this.actionStartedAt = Date.now();
|
|
2351
|
+
const lists = listsFor(this.policy, session.app);
|
|
2352
|
+
if (!dryRun && lists.confirm.length === 0 && lists.block.length === 0)
|
|
2353
|
+
return perform();
|
|
2354
|
+
const stale = this.staleRef(params);
|
|
2355
|
+
if (stale !== null)
|
|
2356
|
+
return stale;
|
|
2357
|
+
if (dryRun) {
|
|
2358
|
+
const resolved = await this.resolveForPolicy(session, method, params);
|
|
2359
|
+
this.logExtra = { dry_run: true };
|
|
2360
|
+
if (resolved.kind === "error") {
|
|
2361
|
+
this.showAction("agent", method, undefined, params, resolved.code, resolved.reason);
|
|
2362
|
+
return fail(resolved.text);
|
|
2363
|
+
}
|
|
2364
|
+
const verdict = this.policyWords(lists, resolved.elements);
|
|
2365
|
+
const shown = resolved.elements[0];
|
|
2366
|
+
if (verdict.kind === "blocked") {
|
|
2367
|
+
this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "blocked");
|
|
2368
|
+
return fail(this.blockedLine(verdict.element, verdict.word));
|
|
2369
|
+
}
|
|
2370
|
+
const would = this.wouldLine(tool, shown, resolved.point, resolved.value);
|
|
2371
|
+
const needs = verdict.kind === "confirm" ? ` It needs a person's OK (policy word ${JSON.stringify(verdict.word)}).` : "";
|
|
2372
|
+
this.showAction("agent", method, shown !== undefined ? { element: shown } : undefined, params);
|
|
2373
|
+
return text([`${would}${needs}`, ...this.takePersonNote()].join("\n"));
|
|
2374
|
+
}
|
|
2375
|
+
let granted = false;
|
|
2376
|
+
const answer = await perform(async () => {
|
|
2377
|
+
const gate = await this.policyGate(session, tool, method, params, lists);
|
|
2378
|
+
if ("stop" in gate)
|
|
2379
|
+
return gate.stop;
|
|
2380
|
+
granted = gate.granted;
|
|
2381
|
+
if (granted)
|
|
2382
|
+
this.logExtra = { confirmed: true };
|
|
2383
|
+
return null;
|
|
2384
|
+
});
|
|
2385
|
+
return granted && answer.isError !== true ? appendNote(answer, "Note: a person allowed this.") : answer;
|
|
2386
|
+
}
|
|
2387
|
+
/**
|
|
2388
|
+
* The policy's verdict on a touch about to happen: stop with an answer (blocked, needs a person's OK, or
|
|
2389
|
+
* the check couldn't be made: fail closed), or go on (`granted` when a person's OK is being used up).
|
|
2390
|
+
*/
|
|
2391
|
+
async policyGate(session, tool, method, params, lists) {
|
|
2392
|
+
if (lists.confirm.length === 0 && lists.block.length === 0)
|
|
2393
|
+
return { granted: false };
|
|
2394
|
+
const resolved = await this.resolveForPolicy(session, method, params);
|
|
2395
|
+
if (resolved.kind === "error") {
|
|
2396
|
+
// Fail closed: what can't be checked isn't touched. The answer is the one the touch would give.
|
|
2397
|
+
this.showAction("agent", method, undefined, params, resolved.code, resolved.reason);
|
|
2398
|
+
return { stop: fail(resolved.text) };
|
|
2399
|
+
}
|
|
2400
|
+
const verdict = this.policyWords(lists, resolved.elements);
|
|
2401
|
+
if (verdict.kind === "blocked") {
|
|
2402
|
+
this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "blocked");
|
|
2403
|
+
return { stop: fail(this.blockedLine(verdict.element, verdict.word)) };
|
|
2404
|
+
}
|
|
2405
|
+
if (verdict.kind === "none")
|
|
2406
|
+
return { granted: false };
|
|
2407
|
+
const label = verdict.element.label ?? verdict.element.text ?? undefined;
|
|
2408
|
+
const caption = captionFor(tool, label ?? undefined, params) ?? tool;
|
|
2409
|
+
// An OK is for this app, this tool and this element only (Tapwright's review, M1).
|
|
2410
|
+
const check = this.confirmations.check(`${session.app.id}|${tool}|${elementKey(verdict.element)}`, { kind: tool, caption, word: verdict.word });
|
|
2411
|
+
if (check.granted)
|
|
2412
|
+
return { granted: true };
|
|
2413
|
+
const name = this.shortName(verdict.element, false);
|
|
2414
|
+
if ("held" in check) {
|
|
2415
|
+
this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "needs_confirmation");
|
|
2416
|
+
return {
|
|
2417
|
+
stop: fail(`Error not_allowed: needs_confirmation: ${name} needs a person's OK (policy word ${JSON.stringify(verdict.word)}), and they declined it just now. Don't ask again by repeating the call: ask your user what they want.`),
|
|
2418
|
+
};
|
|
2419
|
+
}
|
|
2420
|
+
const where = this.options.canConfirm === false
|
|
2421
|
+
? "The live view is off, so nobody can allow it here; ask your user."
|
|
2422
|
+
: "The live view asks them now; when they allow it, make the same call again. If nobody is watching, ask your user.";
|
|
2423
|
+
this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "needs_confirmation");
|
|
2424
|
+
return {
|
|
2425
|
+
stop: fail(`Error not_allowed: needs_confirmation: ${this.shortName(verdict.element, false)} needs a person's OK (policy word ${JSON.stringify(verdict.word)}). ${where}${check.suffix}`),
|
|
2426
|
+
};
|
|
2427
|
+
}
|
|
2428
|
+
/** Which policy word, if any, the elements the target could mean carry: a block word first, then a confirm word. */
|
|
2429
|
+
policyWords(lists, elements) {
|
|
2430
|
+
for (const element of elements) {
|
|
2431
|
+
const word = matchWord(lists.block, element);
|
|
2432
|
+
if (word !== null)
|
|
2433
|
+
return { kind: "blocked", element, word };
|
|
2434
|
+
}
|
|
2435
|
+
for (const element of elements) {
|
|
2436
|
+
const word = matchWord(lists.confirm, element);
|
|
2437
|
+
if (word !== null)
|
|
2438
|
+
return { kind: "confirm", element, word };
|
|
2439
|
+
}
|
|
2440
|
+
return { kind: "none" };
|
|
2441
|
+
}
|
|
2442
|
+
blockedLine(element, word) {
|
|
2443
|
+
return `Error not_allowed: blocked: ${this.shortName(element, false)} is blocked by the policy (${JSON.stringify(word)}); don't try another way.`;
|
|
2444
|
+
}
|
|
2445
|
+
/** A dry run the app answers itself (scroll, back, swipe): its checks, nothing touched, one line. */
|
|
2446
|
+
async dryCall(method, params, line) {
|
|
2447
|
+
const session = this.host.current();
|
|
2448
|
+
if (session === undefined)
|
|
2449
|
+
return this.notConnected();
|
|
2450
|
+
if (!session.capabilities.includes("dryRun")) {
|
|
2451
|
+
return fail("Error unsupported: this app's Tapline is too old for dry runs; update react-native-tapline.");
|
|
2452
|
+
}
|
|
2453
|
+
const stale = this.staleRef(params);
|
|
2454
|
+
if (stale !== null)
|
|
2455
|
+
return stale;
|
|
2456
|
+
this.actionStartedAt = Date.now();
|
|
2457
|
+
let result;
|
|
2458
|
+
try {
|
|
2459
|
+
result = await this.host.request(method, { ...params, dryRun: true }, { sessionId: session.id });
|
|
2460
|
+
}
|
|
2461
|
+
catch (error) {
|
|
2462
|
+
this.logExtra = { dry_run: true };
|
|
2463
|
+
this.showAction("agent", method, undefined, params, error instanceof TaplineError ? error.code : "internal", error instanceof TaplineError ? reasonOf(error.code, error.message) : undefined);
|
|
2464
|
+
return fail(describeError(error));
|
|
2465
|
+
}
|
|
2466
|
+
if (typeof result !== "object" || result === null || result.dryRun !== true) {
|
|
2467
|
+
return fail("Error internal: the app didn't answer the dry run.");
|
|
2468
|
+
}
|
|
2469
|
+
this.logExtra = { dry_run: true };
|
|
2470
|
+
this.showAction("agent", method, result, params);
|
|
2471
|
+
return text([line(result), ...this.takePersonNote()].join("\n"));
|
|
2472
|
+
}
|
|
2473
|
+
/** A dry run of a tool the policy doesn't guard (type_text): the app's checks, nothing touched. */
|
|
2474
|
+
async dryRun(tool, method, params) {
|
|
2475
|
+
const session = this.host.current();
|
|
2476
|
+
if (session === undefined)
|
|
2477
|
+
return this.notConnected();
|
|
2478
|
+
const stale = this.staleRef(params);
|
|
2479
|
+
if (stale !== null)
|
|
2480
|
+
return stale;
|
|
2481
|
+
const resolved = await this.resolveForPolicy(session, method, params);
|
|
2482
|
+
this.logExtra = { dry_run: true };
|
|
2483
|
+
if (resolved.kind === "error") {
|
|
2484
|
+
this.showAction("agent", method, undefined, params, resolved.code, resolved.reason);
|
|
2485
|
+
return fail(resolved.text);
|
|
2486
|
+
}
|
|
2487
|
+
this.showAction("agent", method, resolved.elements[0] !== undefined ? { element: resolved.elements[0] } : undefined, params);
|
|
2488
|
+
return text([this.wouldLine(tool, resolved.elements[0], resolved.point, resolved.value), ...this.takePersonNote()].join("\n"));
|
|
2489
|
+
}
|
|
2490
|
+
/**
|
|
2491
|
+
* What a touch would act on. Apps that know dry runs resolve it themselves, with every check; older
|
|
2492
|
+
* apps only give their element list, and every element the target could mean is checked.
|
|
2493
|
+
*/
|
|
2494
|
+
async resolveForPolicy(session, method, params) {
|
|
2495
|
+
if (session.capabilities.includes("dryRun")) {
|
|
2496
|
+
let raw;
|
|
2497
|
+
try {
|
|
2498
|
+
raw = await this.host.request(method, { ...params, dryRun: true }, { sessionId: session.id });
|
|
2499
|
+
}
|
|
2500
|
+
catch (error) {
|
|
2501
|
+
const reason = error instanceof TaplineError ? reasonOf(error.code, error.message) : undefined;
|
|
2502
|
+
return {
|
|
2503
|
+
kind: "error",
|
|
2504
|
+
text: describeError(error),
|
|
2505
|
+
code: error instanceof TaplineError ? error.code : "internal",
|
|
2506
|
+
...(reason !== undefined ? { reason } : {}),
|
|
2507
|
+
};
|
|
2508
|
+
}
|
|
2509
|
+
const parsed = dryRunResultSchema.safeParse(raw);
|
|
2510
|
+
if (!parsed.success)
|
|
2511
|
+
return { kind: "error", text: "Error internal: the app didn't answer the dry run.", code: "internal" };
|
|
2512
|
+
return {
|
|
2513
|
+
kind: "ok",
|
|
2514
|
+
elements: parsed.data.element ? [parsed.data.element] : [],
|
|
2515
|
+
...(parsed.data.point !== undefined ? { point: parsed.data.point } : {}),
|
|
2516
|
+
...(parsed.data.value !== undefined ? { value: parsed.data.value } : {}),
|
|
2517
|
+
};
|
|
2518
|
+
}
|
|
2519
|
+
// An app without dry runs: a swipe is checked by what lies under its start.
|
|
2520
|
+
const target = method === "swipe" ? { x: params["x0"], y: params["y0"] } : params["target"];
|
|
2521
|
+
if (method === "type" || target === undefined) {
|
|
2522
|
+
return { kind: "error", text: "Error unsupported: this app's Tapline is too old for dry runs; update react-native-tapline.", code: "unsupported" };
|
|
2523
|
+
}
|
|
2524
|
+
let raw;
|
|
2525
|
+
try {
|
|
2526
|
+
raw = await this.host.request("tree", {}, { sessionId: session.id });
|
|
2527
|
+
}
|
|
2528
|
+
catch (error) {
|
|
2529
|
+
return { kind: "error", text: describeError(error), code: error instanceof TaplineError ? error.code : "internal" };
|
|
2530
|
+
}
|
|
2531
|
+
const tree = treeResultSchema.safeParse(raw);
|
|
2532
|
+
if (!tree.success)
|
|
2533
|
+
return { kind: "error", text: "Error internal: the app returned an element list the host could not read.", code: "internal" };
|
|
2534
|
+
const elements = candidatesFor(target, tree.data.elements);
|
|
2535
|
+
if (elements.length === 0 && method !== "swipe") {
|
|
2536
|
+
return { kind: "error", text: "Error not_found: nothing on screen matches that target.", code: "not_found" };
|
|
2537
|
+
}
|
|
2538
|
+
const first = elements[0];
|
|
2539
|
+
const frame = elements.length === 1 && first !== undefined ? first.frame : null;
|
|
2540
|
+
return {
|
|
2541
|
+
kind: "ok",
|
|
2542
|
+
elements,
|
|
2543
|
+
...(frame ? { point: { x: Math.round(frame.x + frame.width / 2), y: Math.round(frame.y + frame.height / 2) } } : {}),
|
|
2544
|
+
};
|
|
2545
|
+
}
|
|
2546
|
+
/** `Would tap e12 button "Delete trip" at 215,640.` */
|
|
2547
|
+
wouldLine(tool, element, point, value) {
|
|
2548
|
+
const name = element !== undefined ? this.shortName(element, true) : "the element";
|
|
2549
|
+
const at = point !== undefined ? ` at ${point.x},${point.y}` : "";
|
|
2550
|
+
switch (tool) {
|
|
2551
|
+
case "long_press":
|
|
2552
|
+
return `Would long-press ${name}${at}.`;
|
|
2553
|
+
case "toggle":
|
|
2554
|
+
return `Would toggle ${name}${typeof value === "boolean" ? ` to ${String(value)}` : ""}${at}.`;
|
|
2555
|
+
case "type_text":
|
|
2556
|
+
return `Would type into ${name}.`;
|
|
2557
|
+
default:
|
|
2558
|
+
return `Would tap ${name}${at}.`;
|
|
2559
|
+
}
|
|
2560
|
+
}
|
|
2561
|
+
/** `e12 button "Delete trip"`: ref (when asked), role and name, nothing else. */
|
|
2562
|
+
shortName(element, withRef) {
|
|
2563
|
+
return formatElement({
|
|
2564
|
+
ref: withRef ? element.ref : "",
|
|
2565
|
+
role: element.role ?? null,
|
|
2566
|
+
type: element.type ?? null,
|
|
2567
|
+
label: element.label ?? null,
|
|
2568
|
+
text: element.text ?? null,
|
|
2569
|
+
textMore: element.textMore ?? null,
|
|
2570
|
+
}, { actions: false });
|
|
2571
|
+
}
|
|
2572
|
+
/** A `{ref}` target after a person used the app: refs may point elsewhere. */
|
|
2573
|
+
staleRef(params) {
|
|
2574
|
+
if (!this.personActed)
|
|
2575
|
+
return null;
|
|
2576
|
+
const target = typeof params === "object" && params !== null ? params.target : undefined;
|
|
2577
|
+
if (typeof target === "object" && target !== null && "ref" in target) {
|
|
2578
|
+
// The error says it already, so the note isn't repeated.
|
|
2579
|
+
this.personNotePending = false;
|
|
2580
|
+
return fail("Error stale: a person used the app since your last look, so refs from that list may point elsewhere. Look again, or target by id, label or text.");
|
|
2581
|
+
}
|
|
2582
|
+
return null;
|
|
2583
|
+
}
|
|
1341
2584
|
tap(args) {
|
|
1342
|
-
|
|
2585
|
+
const params = { target: toWireTarget(withIndex(args.target, args.index)), ...(args.force === true ? { force: true } : {}) };
|
|
2586
|
+
return this.guarded("tap", "press", params, args.dry_run === true, (preflight) => this.act("press", params, (r) => `Tapped ${this.elementName(r)}.`, args.look, args.screenshot, undefined, undefined, preflight));
|
|
1343
2587
|
}
|
|
1344
2588
|
/** `seconds` is accepted for the shared interface; Tapline calls the handler, so it doesn't hold. */
|
|
1345
2589
|
longPress(args) {
|
|
1346
|
-
|
|
2590
|
+
const params = { target: toWireTarget(withIndex(args.target, args.index)), ...(args.force === true ? { force: true } : {}) };
|
|
2591
|
+
return this.guarded("long_press", "longPress", params, args.dry_run === true, (preflight) => this.act("longPress", params, (r) => `Long-pressed ${this.elementName(r)}.`, args.look, args.screenshot, undefined, undefined, preflight));
|
|
1347
2592
|
}
|
|
1348
2593
|
typeText(args) {
|
|
1349
2594
|
const params = {
|
|
@@ -1352,13 +2597,18 @@ export class TaplineTools {
|
|
|
1352
2597
|
submit: args.submit ?? false,
|
|
1353
2598
|
};
|
|
1354
2599
|
if (args.target !== undefined)
|
|
1355
|
-
params["target"] = toWireTarget(args.target);
|
|
2600
|
+
params["target"] = toWireTarget(withIndex(args.target, args.index));
|
|
1356
2601
|
if (args.force === true)
|
|
1357
2602
|
params["force"] = true;
|
|
2603
|
+
// Typing isn't guarded by the policy (§11.1: the form's button is); a dry run still checks the field.
|
|
2604
|
+
if (args.dry_run === true)
|
|
2605
|
+
return this.dryRun("type_text", "type", params);
|
|
1358
2606
|
return this.act("type", params, (r) => {
|
|
1359
2607
|
const value = typeof r === "object" && r !== null && "value" in r ? r.value : undefined;
|
|
1360
|
-
|
|
1361
|
-
|
|
2608
|
+
// AGENT-TOOLS 1.8 §3.3: a secure field's value is never shown, not even as bullets in this line
|
|
2609
|
+
const secure = typeof r === "object" && r !== null && "secure" in r && r.secure === true;
|
|
2610
|
+
const shown = secure ? " The value is hidden (secure field)." : typeof value === "string" ? ` Value is now ${JSON.stringify(value)}.` : "";
|
|
2611
|
+
return `Typed into ${this.elementName(r, "element", shown !== "")}${args.submit ? " and submitted" : ""}.${shown}`;
|
|
1362
2612
|
}, args.look, args.screenshot);
|
|
1363
2613
|
}
|
|
1364
2614
|
scroll(args) {
|
|
@@ -1375,6 +2625,13 @@ export class TaplineTools {
|
|
|
1375
2625
|
const flag = (r, key) => typeof r === "object" && r !== null && key in r ? r[key] : undefined;
|
|
1376
2626
|
// The element was on screen and clear already: nothing was scrolled, and the first line says so.
|
|
1377
2627
|
const alreadyThere = (r) => toTarget && flag(r, "unchanged") === true;
|
|
2628
|
+
if (args.dry_run === true) {
|
|
2629
|
+
return this.dryCall("scroll", params, (r) => {
|
|
2630
|
+
if (toTarget)
|
|
2631
|
+
return `Would scroll ${this.elementName(r)} to ${this.elementName(r, "target")}.`;
|
|
2632
|
+
return `Would scroll ${this.elementName(r)} ${args.to !== undefined ? `to ${args.to}` : (args.direction ?? "down")}.`;
|
|
2633
|
+
});
|
|
2634
|
+
}
|
|
1378
2635
|
const atEndAlready = (r) => (args.to === "start" || args.to === "end") && flag(r, "moved") === false;
|
|
1379
2636
|
return this.act("scroll", params, (r) => {
|
|
1380
2637
|
if (alreadyThere(r))
|
|
@@ -1410,19 +2667,19 @@ export class TaplineTools {
|
|
|
1410
2667
|
});
|
|
1411
2668
|
}
|
|
1412
2669
|
toggle(args) {
|
|
1413
|
-
const params = { target: toWireTarget(args.target) };
|
|
2670
|
+
const params = { target: toWireTarget(withIndex(args.target, args.index)) };
|
|
1414
2671
|
if (args.value !== undefined)
|
|
1415
2672
|
params["value"] = args.value;
|
|
1416
2673
|
if (args.force === true)
|
|
1417
2674
|
params["force"] = true;
|
|
1418
2675
|
// The switch already had the asked value: the app touched nothing, and that line is the whole answer.
|
|
1419
2676
|
const unchanged = (r) => typeof r === "object" && r !== null && "unchanged" in r && r.unchanged === true;
|
|
1420
|
-
return this.act("toggle", params, (r) => {
|
|
2677
|
+
return this.guarded("toggle", "toggle", params, args.dry_run === true, (preflight) => this.act("toggle", params, (r) => {
|
|
1421
2678
|
const value = typeof r === "object" && r !== null && "value" in r ? r.value : undefined;
|
|
1422
2679
|
if (unchanged(r))
|
|
1423
2680
|
return `${this.elementName(r)} is already ${JSON.stringify(value)}.`;
|
|
1424
2681
|
return `Toggled ${this.elementName(r)}${value !== undefined ? ` to ${JSON.stringify(value)}` : ""}.`;
|
|
1425
|
-
}, args.look, args.screenshot, undefined, unchanged);
|
|
2682
|
+
}, args.look, args.screenshot, undefined, unchanged, preflight));
|
|
1426
2683
|
}
|
|
1427
2684
|
async appCommand(args) {
|
|
1428
2685
|
if (args.name === undefined || args.name.trim() === "") {
|
|
@@ -1474,12 +2731,30 @@ export class TaplineTools {
|
|
|
1474
2731
|
const params = { x0: args.x0, y0: args.y0, x1: args.x1, y1: args.y1 };
|
|
1475
2732
|
if (args.seconds !== undefined)
|
|
1476
2733
|
params["seconds"] = args.seconds;
|
|
1477
|
-
|
|
2734
|
+
if (args.dry_run === true) {
|
|
2735
|
+
return this.dryCall("swipe", params, (r) => {
|
|
2736
|
+
const on = this.elementName(r);
|
|
2737
|
+
return `Would swipe from (${args.x0}, ${args.y0}) to (${args.x1}, ${args.y1})${on === "element" ? "" : ` starting on ${on}`}.`;
|
|
2738
|
+
});
|
|
2739
|
+
}
|
|
2740
|
+
// A swipe that moves less than 10 pt is a touch where it starts: the policy checks it (AGENT-TOOLS 1.8 §11.1).
|
|
2741
|
+
const short = Math.hypot(args.x1 - args.x0, args.y1 - args.y0) < 10;
|
|
2742
|
+
const perform = (preflight) => this.act("swipe", params, (r) => {
|
|
1478
2743
|
const on = this.elementName(r);
|
|
1479
2744
|
return `Swiped from (${args.x0}, ${args.y0}) to (${args.x1}, ${args.y1})${on === "element" ? "" : ` on ${on}`}.`;
|
|
1480
|
-
}, args.look, args.screenshot);
|
|
2745
|
+
}, args.look, args.screenshot, undefined, undefined, preflight);
|
|
2746
|
+
return short ? this.guarded("swipe", "swipe", params, false, perform) : perform();
|
|
1481
2747
|
}
|
|
1482
2748
|
back(args) {
|
|
2749
|
+
if (args.dry_run === true) {
|
|
2750
|
+
return this.dryCall("back", {}, (r) => {
|
|
2751
|
+
if (typeof r === "object" && r !== null && "closed" in r && r.closed === "sheet")
|
|
2752
|
+
return "Would close the sheet.";
|
|
2753
|
+
const route = typeof r === "object" && r !== null && "route" in r ? r.route : undefined;
|
|
2754
|
+
const from = Array.isArray(route) && route.every((name) => typeof name === "string") ? ` from ${route.join(" › ")}` : "";
|
|
2755
|
+
return `Would go back${from}.`;
|
|
2756
|
+
});
|
|
2757
|
+
}
|
|
1483
2758
|
return this.act("back", {}, (r) => {
|
|
1484
2759
|
if (typeof r === "object" && r !== null && "closed" in r && r.closed === "sheet")
|
|
1485
2760
|
return "Closed the sheet.";
|
|
@@ -1488,8 +2763,16 @@ export class TaplineTools {
|
|
|
1488
2763
|
return `Went back${from}.`;
|
|
1489
2764
|
}, args.look, args.screenshot);
|
|
1490
2765
|
}
|
|
1491
|
-
openUrl(args) {
|
|
1492
|
-
|
|
2766
|
+
async openUrl(args) {
|
|
2767
|
+
const answer = await this.act("openURL", { url: args.url }, () => `Opened ${args.url}.`, args.look, args.screenshot);
|
|
2768
|
+
// The app's own forget link unpairs it, so it drops the connection before answering: that is the link working.
|
|
2769
|
+
if (answer.isError === true && /(^|\/)tapline\/forget\/?$/.test(args.url.replace(/[?#].*$/, ""))) {
|
|
2770
|
+
const said = answer.content.map((c) => (c.type === "text" ? c.text : "")).join("\n");
|
|
2771
|
+
if (/^Error (reconnecting|not_connected)\b/.test(said)) {
|
|
2772
|
+
return text("Unpaired: the app forgot this computer and disconnected. To connect it again, call `pair`.");
|
|
2773
|
+
}
|
|
2774
|
+
}
|
|
2775
|
+
return answer;
|
|
1493
2776
|
}
|
|
1494
2777
|
async logs(args) {
|
|
1495
2778
|
const session = this.host.current();
|