tapline-host 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/actionlog.d.ts +43 -0
  2. package/dist/actionlog.js +62 -0
  3. package/dist/actionlog.js.map +1 -0
  4. package/dist/audit.d.ts +44 -0
  5. package/dist/audit.js +60 -0
  6. package/dist/audit.js.map +1 -0
  7. package/dist/cli.js +82 -17
  8. package/dist/cli.js.map +1 -1
  9. package/dist/confirm.d.ts +69 -0
  10. package/dist/confirm.js +120 -0
  11. package/dist/confirm.js.map +1 -0
  12. package/dist/format.d.ts +8 -1
  13. package/dist/format.js +53 -11
  14. package/dist/format.js.map +1 -1
  15. package/dist/images.d.ts +7 -0
  16. package/dist/images.js +10 -0
  17. package/dist/images.js.map +1 -1
  18. package/dist/init.d.ts +59 -0
  19. package/dist/init.js +475 -0
  20. package/dist/init.js.map +1 -0
  21. package/dist/inspect.d.ts +75 -0
  22. package/dist/inspect.js +127 -0
  23. package/dist/inspect.js.map +1 -0
  24. package/dist/mcp.d.ts +4 -1
  25. package/dist/mcp.js +56 -2
  26. package/dist/mcp.js.map +1 -1
  27. package/dist/policy.d.ts +42 -0
  28. package/dist/policy.js +74 -0
  29. package/dist/policy.js.map +1 -0
  30. package/dist/run.d.ts +53 -0
  31. package/dist/run.js +97 -0
  32. package/dist/run.js.map +1 -0
  33. package/dist/script.d.ts +76 -0
  34. package/dist/script.js +169 -0
  35. package/dist/script.js.map +1 -0
  36. package/dist/server.d.ts +39 -0
  37. package/dist/server.js +120 -0
  38. package/dist/server.js.map +1 -1
  39. package/dist/tool-text-data.d.ts +25 -8
  40. package/dist/tool-text-data.js +25 -8
  41. package/dist/tool-text-data.js.map +1 -1
  42. package/dist/tool-text.d.ts +1 -1
  43. package/dist/tools.d.ts +221 -7
  44. package/dist/tools.js +1451 -168
  45. package/dist/tools.js.map +1 -1
  46. package/dist/viewer-page.js +1 -1
  47. package/dist/viewer-page.js.map +1 -1
  48. package/dist/viewer.d.ts +3 -0
  49. package/dist/viewer.js +21 -4
  50. package/dist/viewer.js.map +1 -1
  51. package/package.json +2 -2
  52. package/skills/tapline/SKILL.md +42 -12
  53. package/skills/tapline/references/setup.md +24 -2
package/dist/tools.js CHANGED
@@ -1,24 +1,31 @@
1
- import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
2
- import { dirname, join } from "node:path";
1
+ import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
2
+ import { basename, dirname, join } from "node:path";
3
3
  import { toBuffer as qrPng } from "qrcode";
4
4
  import { z } from "zod";
5
5
  import { annotateScreenshot } from "./annotate.js";
6
6
  import { agentKind, captionFor, personKind, reasonOf } from "./captions.js";
7
- import { composeFilmstrip, diffImages } from "./images.js";
7
+ import { composeFilmstrip, diffImages, imageSize } from "./images.js";
8
8
  import { alertNotes, describeElement, formatChanges, formatElement, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
9
+ import { formatInspect, inspectResultSchema } from "./inspect.js";
10
+ import { AUDIT_TYPES, auditResultSchema, auditTitle, formatAudit } from "./audit.js";
11
+ import { ActionLog, loggedTarget } from "./actionlog.js";
12
+ import { Confirmations } from "./confirm.js";
13
+ import { DEFAULT_POLICY, elementKey, listsFor, matchWord } from "./policy.js";
14
+ import { readScript, refStep, scriptSteps } from "./script.js";
15
+ import { EXPECT_TIMEOUT_S, MAX_STEPS, RUN_TIMEOUT_MS, RUN_TOOLS, describeRunStep, expectFailure, parseError, stepNotes, stepSchema, } from "./run.js";
9
16
  import { PAIR_CODE_TTL_MS, forgetLink, pairingLink } from "./pairing.js";
10
17
  import { elementSchema, logEntrySchema, screenshotResultSchema, treeResultSchema, } from "./protocol.js";
11
18
  import { TaplineError } from "./server.js";
12
19
  // ---- input schemas ----------------------------------------------------------
13
20
  export const targetSchema = z
14
21
  .object({
15
- ref: z.string().optional().describe('Element ref from your latest list, e.g. "e12"'),
16
- id: z.string().optional().describe("Exact identifier (React Native testID, the iOS accessibility identifier)"),
17
- label: z.string().optional().describe("Accessibility label, exact, ignoring case"),
18
- text: z.string().optional().describe("Visible text: an exact match first (ignoring case), then a substring"),
19
- index: z.number().int().min(0).optional().describe("With id, label or text: pick the Nth match (0-based, reading order)"),
20
- x: z.number().optional().describe("X in points; use together with y"),
21
- y: z.number().optional().describe("Y in points; use together with x"),
22
+ ref: z.string().optional().describe('From your latest list, e.g. "e12"'),
23
+ id: z.string().optional().describe("The testID, exact"),
24
+ label: z.string().optional().describe("Accessibility label, exact"),
25
+ text: z.string().optional().describe("Visible text: exact first, then part"),
26
+ index: z.number().int().min(0).optional().describe("The Nth match of id, label or text (0-based)"),
27
+ x: z.number().optional().describe("Points, with y"),
28
+ y: z.number().optional().describe("Points, with x"),
22
29
  })
23
30
  .strict()
24
31
  .superRefine((value, ctx) => {
@@ -40,22 +47,38 @@ export const targetSchema = z
40
47
  ctx.addIssue({ code: "custom", message: "index works with id, label or text." });
41
48
  }
42
49
  });
43
- const targetField = targetSchema.describe("Which element: exactly one of {ref}, {id}, {label}, {text} or {x, y}; add index to pick among several id, label or text matches. Prefer ref or id from your latest list.");
50
+ const rectSchema = z
51
+ .object({
52
+ x: z.number().describe("Points"),
53
+ y: z.number().describe("Points"),
54
+ width: z.number().positive().describe("Points"),
55
+ height: z.number().positive().describe("Points"),
56
+ })
57
+ .strict();
58
+ const ignoreListSchema = z.array(z.union([rectSchema, targetSchema]));
59
+ const targetField = targetSchema.describe("Exactly one of ref, id, label, text or x and y");
44
60
  const lookField = z
45
61
  .boolean()
46
62
  .optional()
47
- .describe("Set false to answer only the first line, with no change list. Call look when you need the screen.");
48
- const screenshotField = z.boolean().optional().describe("Add a screenshot to the answer (default false)");
63
+ .describe("false: only the first line; the next answer's changes then cover this action too");
64
+ const screenshotField = z.boolean().optional().describe("Add a screenshot (default false)");
65
+ const dryRunField = z
66
+ .boolean()
67
+ .optional()
68
+ .describe("true: run every check and say what would happen; touch nothing");
69
+ /** `index` written beside `target` rather than in it: agents do that, and it means the same. */
70
+ const indexField = z.number().int().min(0).optional().describe("Same as target.index");
49
71
  const forceField = z
50
72
  .boolean()
51
73
  .optional()
52
- .describe("Skip the off-screen, covered, under-the-tab-bar and behind-an-alert checks (never disabled). Default false; prefer scrolling or dismissing what covers it, since a person would have to.");
74
+ .describe("Skip the reach checks (never disabled), only when you've decided a refusal is wrong");
53
75
  export const toolShapes = {
54
76
  apps: {},
55
77
  open_app: { app: z.string().describe("The app's session id, app id or name, as apps lists them") },
56
78
  pair: {
57
79
  scheme: z
58
80
  .string()
81
+ .optional()
59
82
  .describe('The app\'s URL scheme ("scheme" in app.json or app.config), e.g. "myapp"; for an app running in Expo Go, the address Expo prints, e.g. "exp://192.168.1.20:8081"'),
60
83
  },
61
84
  look: {
@@ -73,9 +96,17 @@ export const toolShapes = {
73
96
  .describe("Draw each actionable element's box and ref (e12) on the screenshot, so whoever views it can name elements by ref"),
74
97
  },
75
98
  check_layout: {},
76
- tap: { target: targetField, force: forceField, look: lookField, screenshot: screenshotField },
99
+ audit: {
100
+ types: z
101
+ .array(z.enum(AUDIT_TYPES))
102
+ .optional()
103
+ .describe("Which checks to run (default all): contrast, hit_region, description, dynamic_type, clipped_text, trait, element_detection, overlap, duplicate_id"),
104
+ },
105
+ inspect: { target: targetField },
106
+ tap: { target: targetField, index: indexField, force: forceField, dry_run: dryRunField, look: lookField, screenshot: screenshotField },
77
107
  long_press: {
78
108
  target: targetField,
109
+ index: indexField,
79
110
  seconds: z
80
111
  .number()
81
112
  .min(0.1)
@@ -83,15 +114,18 @@ export const toolShapes = {
83
114
  .optional()
84
115
  .describe("How long to hold (default 1.0). Tapline calls the handler, so it doesn't wait"),
85
116
  force: forceField,
117
+ dry_run: dryRunField,
86
118
  look: lookField,
87
119
  screenshot: screenshotField,
88
120
  },
89
121
  type_text: {
90
122
  text: z.string().describe("Text to enter"),
91
123
  target: targetField.optional().describe("Input to type into; default: the focused input, else the only input on screen (else ambiguous; none at all is not_found)"),
124
+ index: indexField,
92
125
  replace: z.boolean().optional().describe("Replace the current text (default true); false appends"),
93
126
  submit: z.boolean().optional().describe("Also submit (like pressing return) after typing (default false)"),
94
127
  force: forceField,
128
+ dry_run: dryRunField,
95
129
  look: lookField,
96
130
  screenshot: screenshotField,
97
131
  },
@@ -105,13 +139,16 @@ export const toolShapes = {
105
139
  target: targetField
106
140
  .optional()
107
141
  .describe('Scroll view to scroll, or an element inside it; default: the largest one on screen. With to: "target", the element to bring onto the screen'),
142
+ dry_run: dryRunField,
108
143
  look: lookField,
109
144
  screenshot: screenshotField,
110
145
  },
111
146
  toggle: {
112
147
  target: targetField,
148
+ index: indexField,
113
149
  value: z.boolean().optional().describe("Desired state; omit to flip"),
114
150
  force: forceField,
151
+ dry_run: dryRunField,
115
152
  look: lookField,
116
153
  screenshot: screenshotField,
117
154
  },
@@ -120,13 +157,14 @@ export const toolShapes = {
120
157
  look: lookField,
121
158
  screenshot: screenshotField,
122
159
  },
123
- back: { look: lookField, screenshot: screenshotField },
160
+ back: { dry_run: dryRunField, look: lookField, screenshot: screenshotField },
124
161
  swipe: {
125
162
  x0: z.number().describe("Where the finger starts, in points from the left"),
126
163
  y0: z.number().describe("Where the finger starts, in points from the top"),
127
164
  x1: z.number().describe("Where it lifts, in points from the left"),
128
165
  y1: z.number().describe("Where it lifts, in points from the top"),
129
166
  seconds: z.number().min(0.05).max(5).optional().describe("How long the drag takes (default 0.35)"),
167
+ dry_run: dryRunField,
130
168
  look: lookField,
131
169
  screenshot: screenshotField,
132
170
  },
@@ -146,6 +184,16 @@ export const toolShapes = {
146
184
  .max(1320)
147
185
  .optional()
148
186
  .describe("Screenshot width in pixels, 160 to 1320 (default 600); keep it the same for a name"),
187
+ threshold: z
188
+ .number()
189
+ .min(0)
190
+ .max(100)
191
+ .optional()
192
+ .describe("The share of the screen, in percent, 0 to 100 (default 0.5), that may change and still match"),
193
+ ignore: z
194
+ .array(z.union([rectSchema, targetSchema]))
195
+ .optional()
196
+ .describe("Painted out on both pictures before comparing (a clock, an avatar): targets {id}, {label}, {text}, {ref}, or a rect {x, y, width, height} in points"),
149
197
  },
150
198
  filmstrip: {
151
199
  frames: z.number().int().min(2).max(9).optional().describe("How many frames (default 6)"),
@@ -155,13 +203,20 @@ export const toolShapes = {
155
203
  },
156
204
  viewer: {},
157
205
  record: {
206
+ start: z.boolean().optional().describe("Start recording: every action that succeeds from now on becomes a step"),
207
+ stop: z.boolean().optional().describe("Stop and save the script (to path, else to the scripts folder under name)"),
208
+ name: z.string().optional().describe("The script's name: letters, digits, '.', '_' and '-'"),
209
+ path: z.string().optional().describe("Where stop saves the script file (optional)"),
158
210
  action: z
159
211
  .enum(["start", "stop", "status", "discard"])
160
- .describe("start a recording (needs name), stop and save it, see the status, or discard it"),
161
- name: z.string().optional().describe("Script name for start: letters, digits, '.', '_' and '-'"),
212
+ .optional()
213
+ .describe("Older form: start, stop, status or discard"),
162
214
  },
163
215
  replay: {
164
- name: z.string().optional().describe("The script to replay; omit to list saved scripts"),
216
+ name: z.string().optional().describe("A saved script's name; omit name and path to list saved scripts"),
217
+ path: z.string().optional().describe("A script file to replay (the shared agent-tools-script format)"),
218
+ look: lookField,
219
+ screenshot: screenshotField,
165
220
  },
166
221
  wait_for: {
167
222
  text: z.string().optional().describe("Wait until this visible text is on screen (or gone)"),
@@ -178,7 +233,93 @@ export const toolShapes = {
178
233
  },
179
234
  reload: { look: lookField },
180
235
  wait: { seconds: z.number().min(0.1).max(30).describe("How long to wait, 0.1 to 30 seconds"), screenshot: screenshotField },
236
+ run: {
237
+ steps: z
238
+ .array(stepSchema)
239
+ .min(1)
240
+ .max(MAX_STEPS)
241
+ .describe('The steps, in order: each is a tool call as an object, {"tool": "tap", "target": {...}, "expect": {...}}. Tools: tap, long_press, type_text, scroll, toggle, back, swipe, open_url, wait, wait_for, app_command, compare_screen (checks, never saves). expect (optional): one of text, id or label, with gone, value and timeout (seconds, default 2); or matches (a reference name, alone or with one of them) with threshold.'),
242
+ trace: z.boolean().optional().describe("Also list each step that ran, one line each"),
243
+ dry_run: z
244
+ .boolean()
245
+ .optional()
246
+ .describe("Check every step's parameters and resolve step 1 (later steps depend on what it does), then touch nothing"),
247
+ look: lookField,
248
+ screenshot: screenshotField,
249
+ },
181
250
  };
251
+ // ---- compare_screen ---------------------------------------------------------
252
+ const DEFAULT_MATCH_THRESHOLD = 0.5;
253
+ const MAX_REGION_LINES = 8;
254
+ function describeIgnore(item) {
255
+ if (item.id !== undefined)
256
+ return `id=${item.id}`;
257
+ if (item.label !== undefined)
258
+ return `label ${JSON.stringify(item.label)}`;
259
+ if (item.text !== undefined)
260
+ return JSON.stringify(item.text);
261
+ if (item.ref !== undefined)
262
+ return item.ref;
263
+ return `${String(item.x)},${String(item.y)}`;
264
+ }
265
+ /** The pixel rectangles to paint out for `ignore`, from a fresh element list; targets not on the screen give a note. */
266
+ /**
267
+ * An element as a {label} target matches it: its accessibility label, else its text, as the app's SDK matches
268
+ * (labelOf in the SDK). The ignore masks, recorded indexes and expectations use the same rule as the targets.
269
+ */
270
+ function labelOfElement(e) {
271
+ return e.label ?? e.text ?? "";
272
+ }
273
+ function ignoreRects(ignore, elements, scale, size) {
274
+ const rects = [];
275
+ const notes = [];
276
+ const add = (x, y, width, height) => {
277
+ const x0 = Math.max(0, Math.floor(x * scale));
278
+ const y0 = Math.max(0, Math.floor(y * scale));
279
+ const x1 = Math.min(size.width, Math.ceil((x + width) * scale));
280
+ const y1 = Math.min(size.height, Math.ceil((y + height) * scale));
281
+ if (x1 > x0 && y1 > y0)
282
+ rects.push({ x: x0, y: y0, width: x1 - x0, height: y1 - y0 });
283
+ };
284
+ for (const item of ignore) {
285
+ if ("width" in item) {
286
+ add(item.x, item.y, item.width, item.height);
287
+ continue;
288
+ }
289
+ let found = [];
290
+ if (item.ref !== undefined)
291
+ found = elements.filter((e) => e.ref === item.ref);
292
+ else if (item.id !== undefined)
293
+ found = elements.filter((e) => e.testID === item.id);
294
+ else if (item.label !== undefined) {
295
+ const wanted = item.label.toLowerCase();
296
+ found = elements.filter((e) => labelOfElement(e).toLowerCase() === wanted);
297
+ }
298
+ else if (item.text !== undefined) {
299
+ const wanted = item.text.toLowerCase();
300
+ found = elements.filter((e) => (e.text ?? "").toLowerCase() === wanted);
301
+ if (found.length === 0)
302
+ found = elements.filter((e) => wanted !== "" && (e.text ?? "").toLowerCase().includes(wanted));
303
+ }
304
+ else if (item.x !== undefined && item.y !== undefined) {
305
+ const px = item.x;
306
+ const py = item.y;
307
+ found = elements
308
+ .filter((e) => e.frame && px >= e.frame.x && px <= e.frame.x + e.frame.width && py >= e.frame.y && py <= e.frame.y + e.frame.height)
309
+ .sort((a, b) => (a.frame?.width ?? 0) * (a.frame?.height ?? 0) - (b.frame?.width ?? 0) * (b.frame?.height ?? 0))
310
+ .slice(0, 1);
311
+ }
312
+ found = found.filter((e) => e.frame !== undefined);
313
+ if (item.index !== undefined)
314
+ found = found.slice(item.index, item.index + 1);
315
+ if (found.length === 0)
316
+ notes.push(`ignore ${describeIgnore(item)} isn't on the screen; skipped.`);
317
+ for (const e of found)
318
+ if (e.frame)
319
+ add(e.frame.x, e.frame.y, e.frame.width, e.frame.height);
320
+ }
321
+ return { rects, notes };
322
+ }
182
323
  /**
183
324
  * Under this many points from its first point, a pointer path is a tap or a hold, not a drag.
184
325
  * Tapwright uses the same number, and the viewer page's pending ring (TAP_SLOP in viewer.html)
@@ -188,6 +329,8 @@ const TAP_SLOP = 10;
188
329
  /** Held at least this long, a tap is a long press. */
189
330
  const LONG_PRESS_MS = 500;
190
331
  const SCRIPT_NAME = /^[A-Za-z0-9][\w.-]{0,60}$/;
332
+ /** The answer to a reference name that isn't one: no slashes, so no way out of the references folder. */
333
+ const BAD_REFERENCE_NAME = "Error bad_params: name the reference with letters, digits, '.', '_' and '-'.";
191
334
  const scriptSchema = z.object({
192
335
  version: z.literal(1),
193
336
  name: z.string(),
@@ -341,6 +484,10 @@ function text(message) {
341
484
  function fail(message) {
342
485
  return { content: [{ type: "text", text: message }], isError: true };
343
486
  }
487
+ /** The target with an `index` that was written beside it moved inside (its own index wins). */
488
+ function withIndex(target, index) {
489
+ return index !== undefined && target.index === undefined ? { ...target, index } : target;
490
+ }
344
491
  function toWireTarget(target) {
345
492
  const withIndex = (t) => target.index !== undefined ? { ...t, index: target.index } : t;
346
493
  if (target.ref !== undefined)
@@ -421,12 +568,75 @@ export function describeError(error) {
421
568
  }
422
569
  return `Error internal: ${error instanceof Error ? error.message : String(error)}`;
423
570
  }
571
+ /** Ends a guarded call inside the agent's turn with the policy's answer (nothing was touched). */
572
+ class PolicyStop extends Error {
573
+ answer;
574
+ constructor(answer) {
575
+ super("stopped by the policy");
576
+ this.answer = answer;
577
+ }
578
+ }
579
+ /** Adds a note line to the end of an answer's first text block. */
580
+ function appendNote(answer, note) {
581
+ const [top, ...rest] = answer.content;
582
+ if (top === undefined || top.type !== "text")
583
+ return answer;
584
+ return { ...answer, content: [{ type: "text", text: [top.text.trimEnd(), note].join("\n") }, ...rest] };
585
+ }
586
+ /** An app's dry-run answer (AGENT-TOOLS 1.8 §11.4). */
587
+ const dryRunResultSchema = z.object({
588
+ element: elementSchema.nullish(),
589
+ dryRun: z.literal(true),
590
+ point: z.object({ x: z.number(), y: z.number() }).optional(),
591
+ value: z.unknown().optional(),
592
+ });
593
+ /** For an app without dry runs: every element on its list the target could mean. */
594
+ function candidatesFor(target, elements) {
595
+ if (typeof target !== "object" || target === null)
596
+ return [];
597
+ const t = target;
598
+ const lower = (value) => (typeof value === "string" ? value.toLowerCase() : "");
599
+ const x = t["x"];
600
+ const y = t["y"];
601
+ let found;
602
+ if (typeof t["ref"] === "string")
603
+ found = elements.filter((e) => e.ref === t["ref"]);
604
+ else if (typeof t["testID"] === "string")
605
+ found = elements.filter((e) => e.testID === t["testID"]);
606
+ else if (typeof t["label"] === "string")
607
+ found = elements.filter((e) => lower(e.label) === lower(t["label"]));
608
+ else if (typeof t["text"] === "string") {
609
+ const want = lower(t["text"]);
610
+ found = elements.filter((e) => lower(e.text).includes(want) || lower(e.label).includes(want));
611
+ }
612
+ else if (typeof x === "number" && typeof y === "number") {
613
+ found = elements.filter((e) => {
614
+ const f = e.frame;
615
+ return f !== null && f !== undefined && x >= f.x && x <= f.x + f.width && y >= f.y && y <= f.y + f.height;
616
+ });
617
+ }
618
+ else
619
+ found = [];
620
+ const index = t["index"];
621
+ const picked = typeof index === "number" ? found[index] : undefined;
622
+ return picked !== undefined ? [picked] : found;
623
+ }
424
624
  // ---- tool handlers ----------------------------------------------------------
425
625
  /** The tool handlers behind the MCP server. They only talk to a TaplineHost, so they are easy to test. */
426
626
  export class TaplineTools {
427
627
  host;
428
628
  options;
429
629
  reloadWaitMs;
630
+ /** The action log (§12), when one is configured. */
631
+ actionLog;
632
+ /** When the action in flight started, for the log's `ms`. */
633
+ actionStartedAt;
634
+ /** Set while `run` runs a step: the log ties the action to its batch. */
635
+ batch;
636
+ /** Extra fields for the next log line (a dry run, a confirmed touch). */
637
+ logExtra;
638
+ confirmations;
639
+ policy;
430
640
  reloadSettleMs;
431
641
  noScreenshot = new Set();
432
642
  shotFiles = [];
@@ -444,6 +654,15 @@ export class TaplineTools {
444
654
  personNotePending = false;
445
655
  recording = null;
446
656
  replaying = false;
657
+ /**
658
+ * While a script replays (AGENT-TOOLS 1.9 §14.3): the references folder next to it (looked in first), where a
659
+ * failed check puts its red picture, and whether the checks save instead of compare (`--update-references`).
660
+ */
661
+ replayContext = null;
662
+ /** Where the latest failed check of the running batch saved its picture. */
663
+ diffSaved = null;
664
+ /** The elements the latest comparison painted its ignore list from: a recording saves its mask from the same list. */
665
+ lastMaskElements;
447
666
  /** The screen size in points from the latest list or overlay, for the live viewer's 0..1 positions. */
448
667
  lastScreen;
449
668
  shotCount = 0;
@@ -452,6 +671,10 @@ export class TaplineTools {
452
671
  this.host = host;
453
672
  this.options = options;
454
673
  this.reloadWaitMs = options.reloadWaitMs ?? 20_000;
674
+ this.confirmations = options.confirmations ?? new Confirmations();
675
+ this.policy = options.policy ?? DEFAULT_POLICY;
676
+ this.actionLog = options.logDir !== undefined ? new ActionLog(options.logDir) : null;
677
+ this.actionLog?.open();
455
678
  this.reloadSettleMs = options.reloadSettleMs ?? 1500;
456
679
  // A finger on the phone itself: the app saw a real touch. It can't be queued like viewer input
457
680
  // (it has already happened), so it only leaves the note and makes `{ref}` targets stale.
@@ -490,15 +713,17 @@ export class TaplineTools {
490
713
  const silent = this.silentDropMessage();
491
714
  lines.push(silent !== undefined
492
715
  ? silent
493
- : state === "background" || state === "inactive"
494
- ? `${last.session.app.name} disconnected at ${when} after it went to the background (the user switched apps or locked the phone, and iOS paused it). Ask the user to open the app again: it reconnects by itself.`
495
- : `${last.session.app.name} disconnected at ${when} (${last.reason}). If the app is still open, it reconnects by itself.`);
716
+ : state === "inactive"
717
+ ? `${last.session.app.name} disconnected at ${when} after iOS put something in front of it (a system prompt such as a password request, a call, or Control Center). Tapline can't see or answer it: ask the user to deal with it on the phone. The app reconnects by itself.`
718
+ : state === "background"
719
+ ? `${last.session.app.name} disconnected at ${when} after it went to the background (the user switched apps or locked the phone, and iOS paused it). Ask the user to open the app again: it reconnects by itself.`
720
+ : `${last.session.app.name} disconnected at ${when} (${last.reason}). If the app is still open, it reconnects by itself.`);
496
721
  }
497
722
  return [
498
723
  "No app is connected to Tapline.",
499
724
  ...lines,
500
725
  `This host listens at ${url} on ${name}.`,
501
- 'To connect a phone, call `pair` with the app\'s URL scheme ("scheme" in app.json, e.g. "myapp") and show the user the QR code. They scan it with the phone\'s camera and tap Connect in the app.',
726
+ 'To connect a phone, call `pair`. A development build loading its code from this computer then asks by itself: the user taps Connect. Other builds: pass the app\'s URL scheme ("scheme" in app.json, e.g. "myapp") and show the user the QR code to scan.',
502
727
  "The phone needs to be on the same Wi-Fi as this computer, and the app must be a build with Tapline turned on, open in the foreground.",
503
728
  "A phone that paired with this host before reconnects by itself while the app is open; one paired with a host on another port (e.g. before a restart) needs `pair` again. Once it connects, retry.",
504
729
  ].join("\n");
@@ -506,6 +731,19 @@ export class TaplineTools {
506
731
  // -- pair
507
732
  async pair(args) {
508
733
  const { url, name } = this.options.connect;
734
+ // Zero-QR pairing (Tapline 1.1): a development build whose JavaScript comes from this computer finds the
735
+ // offer itself and asks its person to connect. Open for 10 minutes, one use, local network only.
736
+ const offer = this.host.openDevOffer(name);
737
+ this.writeOffer(offer);
738
+ const devLine = "A development build that loads its JavaScript from this computer (Metro) on the same Wi-Fi asks by itself within a few seconds: the user only taps Connect, no QR code (open for 10 minutes, once).";
739
+ if (args.scheme === undefined || args.scheme.trim() === "") {
740
+ return text([
741
+ `Pairing is open for development builds for 10 minutes (${name}).`,
742
+ devLine,
743
+ "Ask the user to open the app on the phone and tap Connect, then call `apps`.",
744
+ "For a TestFlight or release build, call pair again with the app's URL scheme to get a QR code.",
745
+ ].join("\n"));
746
+ }
509
747
  let link;
510
748
  let forget;
511
749
  try {
@@ -540,6 +778,7 @@ export class TaplineTools {
540
778
  "",
541
779
  `Show the user the QR code (attached).${saved} They scan it with the phone's camera, open the app and tap Connect.`,
542
780
  "The phone needs to be on the same Wi-Fi as this computer, and the app must be a build with Tapline turned on.",
781
+ devLine,
543
782
  "Once paired, the app reconnects to this host (this address and port) by itself whenever it is open; call `apps` to check.",
544
783
  `To unpair the phone, open ${forget} on it.`,
545
784
  ];
@@ -550,6 +789,51 @@ export class TaplineTools {
550
789
  ],
551
790
  };
552
791
  }
792
+ /** `<offerDir>/<port>.json` (0600): the open offer for the Metro helper. Never fails the pair call. */
793
+ writeOffer(offer) {
794
+ const dir = this.options.offerDir;
795
+ const port = this.host.boundPort();
796
+ if (dir === undefined || port === undefined)
797
+ return;
798
+ try {
799
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
800
+ const file = join(dir, `${port}.json`);
801
+ writeFileSync(file, JSON.stringify({ code: offer.code, name: offer.name, port, until: offer.until }), { mode: 0o600 });
802
+ this.host.setDevOfferFile(file);
803
+ const remove = () => {
804
+ try {
805
+ // only our own offer: a newer pair call has written its own
806
+ if (existsSync(file) && readFileSync(file, "utf8").includes(offer.code))
807
+ rmSync(file, { force: true });
808
+ }
809
+ catch {
810
+ // gone already
811
+ }
812
+ };
813
+ // Used: the code works once, so Metro stops handing it out at once.
814
+ const onConnect = () => {
815
+ if (this.host.currentDevOffer()?.code === offer.code)
816
+ return;
817
+ this.host.off("connect", onConnect);
818
+ remove();
819
+ };
820
+ this.host.on("connect", onConnect);
821
+ setTimeout(() => {
822
+ this.host.off("connect", onConnect);
823
+ try {
824
+ // only our own offer: a newer pair call has written its own
825
+ if (existsSync(file) && readFileSync(file, "utf8").includes(offer.code))
826
+ rmSync(file, { force: true });
827
+ }
828
+ catch {
829
+ // gone already
830
+ }
831
+ }, Math.max(0, offer.until - Date.now())).unref();
832
+ }
833
+ catch {
834
+ // the host's own endpoint still has it
835
+ }
836
+ }
553
837
  // -- apps
554
838
  apps() {
555
839
  const sessions = this.host.sessions();
@@ -602,6 +886,258 @@ export class TaplineTools {
602
886
  const parts = await this.lookParts(session, args.screenshot ?? true, args.width ?? DEFAULT_SCREENSHOT_WIDTH, false, args.labels === true);
603
887
  return { content: parts.content, ...(parts.isError ? { isError: true } : {}) };
604
888
  }
889
+ // -- run (AGENT-TOOLS 1.8 §9): a batch of steps, answered once
890
+ async run(args) {
891
+ const session = this.host.current();
892
+ if (session === undefined)
893
+ return this.notConnected();
894
+ const steps = args.steps;
895
+ for (const [i, step] of steps.entries()) {
896
+ if (!RUN_TOOLS.includes(step.tool)) {
897
+ return fail(`Error bad_params: step ${i + 1}: "${step.tool}" can't be a step (steps act; use it on its own call). Steps can be: ${RUN_TOOLS.join(", ")}.`);
898
+ }
899
+ const shape = toolShapes[step.tool];
900
+ const { tool: _tool, expect: _expect, route: _route, ...params } = step;
901
+ if (step.tool === "compare_screen" && params.update !== undefined) {
902
+ return fail(`Error bad_params: step ${i + 1}: update can't be used in run (a step never saves a reference)`);
903
+ }
904
+ const parsed = z.object(shape).strict().safeParse(params);
905
+ if (!parsed.success) {
906
+ return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): ${parsed.error.issues.map((issue) => `${issue.path.join(".") || "params"}: ${issue.message}`).join("; ")}`);
907
+ }
908
+ // Every reference name is checked before any step runs: a name with a path in it never reaches the disk.
909
+ const names = [step.tool === "compare_screen" ? params.name : undefined, step.expect?.matches];
910
+ if (names.some((name) => typeof name === "string" && !SCRIPT_NAME.test(name.trim()))) {
911
+ return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): name the reference with letters, digits, '.', '_' and '-'.`);
912
+ }
913
+ if (step.expect?.ignore !== undefined) {
914
+ const ignored = ignoreListSchema.safeParse(step.expect.ignore);
915
+ if (!ignored.success) {
916
+ return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): expect.ignore: ${ignored.error.issues.map((issue) => issue.message).join("; ")}`);
917
+ }
918
+ }
919
+ }
920
+ this.diffSaved = null;
921
+ if (args.dry_run === true)
922
+ return this.dryRunBatch(steps);
923
+ const before = this.unlistedActions.get(session.id) ?? 0;
924
+ const started = Date.now();
925
+ const batchId = `b${started.toString(36)}`;
926
+ const notes = [];
927
+ const trace = [];
928
+ let ran = 0;
929
+ let stop = null;
930
+ for (const [i, step] of steps.entries()) {
931
+ if (Date.now() - started > RUN_TIMEOUT_MS) {
932
+ stop = { index: i, code: "timeout", message: `the batch ran longer than ${RUN_TIMEOUT_MS / 1000} s`, rest: [] };
933
+ break;
934
+ }
935
+ const at = Date.now();
936
+ this.batch = { id: batchId, step: i + 1 };
937
+ const answer = await this.runStep(step).finally(() => {
938
+ this.batch = undefined;
939
+ });
940
+ const said = answer.content.map((block) => (block.type === "text" ? block.text : "")).join("\n").trim();
941
+ if (answer.isError === true) {
942
+ stop = { index: i, ...parseError(said) };
943
+ break;
944
+ }
945
+ notes.push(...stepNotes(said, i + 1));
946
+ if (step.expect !== undefined) {
947
+ const missed = await this.expectMissed(step.expect);
948
+ if (missed !== null) {
949
+ stop = { index: i, code: "expect_failed", message: `its action ran, but ${missed}`, rest: [] };
950
+ break;
951
+ }
952
+ // A picture check that held is kept in a recording, as compare_screen's is: right after the action, the
953
+ // script makes it that step's `matches` again (AGENT-TOOLS 1.9 §14.2).
954
+ if (step.expect.matches !== undefined) {
955
+ const recorded = await this.recordCompare({
956
+ name: step.expect.matches,
957
+ ...(step.expect.threshold !== undefined ? { threshold: step.expect.threshold } : {}),
958
+ ...(step.expect.ignore !== undefined ? { ignore: ignoreListSchema.parse(step.expect.ignore) } : {}),
959
+ });
960
+ if (recorded !== null) {
961
+ stop = { index: i, ...parseError(recorded) };
962
+ break;
963
+ }
964
+ }
965
+ }
966
+ const route = step.route;
967
+ if (Array.isArray(route) && route.every((part) => typeof part === "string")) {
968
+ const now = await this.currentRoute();
969
+ if (now !== null && now.join(" \u203a ") !== route.join(" \u203a ")) {
970
+ stop = { index: i, code: "expect_failed", message: `its action ran, but it ended on ${now.join(" \u203a ")}; the recording ended on ${route.join(" \u203a ")}`, rest: [] };
971
+ break;
972
+ }
973
+ }
974
+ ran++;
975
+ if (args.trace === true)
976
+ trace.push(` ${i + 1}. ${said.split("\n")[0] ?? ""} (${((Date.now() - at) / 1000).toFixed(1)} s)`);
977
+ }
978
+ // The batch is one call: the changes below are measured from the list the agent had before it.
979
+ if (before > 0)
980
+ this.unlistedActions.set(session.id, before);
981
+ else
982
+ this.unlistedActions.delete(session.id);
983
+ const total = steps.length;
984
+ const failed = stop !== null;
985
+ const head = stop === null
986
+ ? `Ran ${total} of ${total} steps in ${((Date.now() - started) / 1000).toFixed(1)} s.`
987
+ : `Error ${stop.code}: Ran ${ran} of ${total} steps; stopped at step ${stop.index + 1} (${describeRunStep(steps[stop.index])}): ${stop.message}`;
988
+ const diffNote = stop !== null && this.diffSaved !== null ? [`Note: the difference is saved at ${this.diffSaved}.`] : [];
989
+ const first = [head, ...trace, ...(stop?.rest ?? []), ...diffNote].join("\n");
990
+ let response;
991
+ const live = this.host.current();
992
+ if (args.look === false || live === undefined) {
993
+ response = text([first, ...notes].join("\n"));
994
+ }
995
+ else {
996
+ response = await this.withLook(live, first, args.screenshot === true);
997
+ if (notes.length > 0) {
998
+ const [top, ...rest] = response.content;
999
+ if (top !== undefined && top.type === "text")
1000
+ response = { ...response, content: [{ type: "text", text: [top.text.trimEnd(), ...notes].join("\n") }, ...rest] };
1001
+ }
1002
+ }
1003
+ return failed ? { ...response, isError: true } : response;
1004
+ }
1005
+ /** `run` with dry_run (§11.4): the steps are checked already; step 1 is resolved, nothing is touched. */
1006
+ async dryRunBatch(steps) {
1007
+ const total = steps.length;
1008
+ const first = steps[0];
1009
+ const head = `Would run ${total} step${total === 1 ? "" : "s"}`;
1010
+ if (["tap", "long_press", "toggle", "type_text", "scroll", "back", "swipe"].includes(first.tool)) {
1011
+ const answer = await this.runStep({ ...first, dry_run: true });
1012
+ const said = answer.content.map((block) => (block.type === "text" ? block.text : "")).join("\n").trim();
1013
+ const [line, ...rest] = said.split("\n");
1014
+ if (answer.isError === true) {
1015
+ const { code, message } = parseError(said);
1016
+ return fail(`Error ${code}: ${head}; step 1 (${describeRunStep(first)}) would fail: ${message}`);
1017
+ }
1018
+ const would = (line ?? "").replace(/^Would /, "would ");
1019
+ return text([`${head}; step 1 ${would}`, ...rest].join("\n"));
1020
+ }
1021
+ return text(`${head}; step 1 would ${describeRunStep(first)}.`);
1022
+ }
1023
+ /** One step of `run`, as its own tool call without a list. */
1024
+ runStep(step) {
1025
+ const { tool, expect: _expect, route: _route, ...rest } = step;
1026
+ const quiet = { ...rest, look: false, screenshot: false };
1027
+ switch (tool) {
1028
+ case "tap":
1029
+ return this.tap(quiet);
1030
+ case "long_press":
1031
+ return this.longPress(quiet);
1032
+ case "type_text":
1033
+ return this.typeText(quiet);
1034
+ case "scroll":
1035
+ return this.scroll(quiet);
1036
+ case "toggle":
1037
+ return this.toggle(quiet);
1038
+ case "back":
1039
+ return this.back(quiet);
1040
+ case "swipe":
1041
+ return this.swipe(quiet);
1042
+ case "open_url":
1043
+ return this.openUrl(quiet);
1044
+ case "wait":
1045
+ return this.wait({ seconds: Number(rest.seconds ?? 0) });
1046
+ case "wait_for":
1047
+ return this.waitFor(quiet);
1048
+ case "app_command":
1049
+ return this.appCommand(quiet);
1050
+ case "compare_screen":
1051
+ return this.compareStep(rest);
1052
+ }
1053
+ }
1054
+ /** The screen's route names now (one cheap read), or null when the app has no router or didn't answer. */
1055
+ async currentRoute() {
1056
+ const session = this.host.current();
1057
+ if (session === undefined)
1058
+ return null;
1059
+ const tree = treeResultSchema.safeParse(await this.host.request("tree", { peek: true }, { sessionId: session.id }).catch(() => null));
1060
+ return tree.success && tree.data.route !== undefined ? tree.data.route.names : null;
1061
+ }
1062
+ /** Null when the expectation holds within its time; else how it failed. */
1063
+ async expectMissed(e) {
1064
+ if (e.text !== undefined || e.id !== undefined || e.label !== undefined) {
1065
+ const missed = await this.textExpectMissed(e);
1066
+ if (missed !== null)
1067
+ return missed;
1068
+ }
1069
+ // A picture check comes last, once the rest held.
1070
+ return e.matches === undefined ? null : this.matchMissed(e.matches, e.threshold ?? DEFAULT_MATCH_THRESHOLD, ignoreListSchema.parse(e.ignore ?? []));
1071
+ }
1072
+ async textExpectMissed(e) {
1073
+ const seconds = e.timeout ?? EXPECT_TIMEOUT_S;
1074
+ const which = e.text !== undefined ? { text: e.text } : e.id !== undefined ? { id: e.id } : { label: e.label };
1075
+ const seen = await this.waitFor({ ...which, ...(e.gone === true ? { gone: true } : {}), timeout: Math.max(0.5, seconds), look: false });
1076
+ if (seen.isError === true)
1077
+ return expectFailure(e, seconds);
1078
+ if (e.value === undefined || e.gone === true)
1079
+ return null;
1080
+ // The element is there: wait for its value (a switch reads "on" or "off").
1081
+ const session = this.host.current();
1082
+ if (session === undefined)
1083
+ return expectFailure(e, seconds);
1084
+ const until = Date.now() + seconds * 1000;
1085
+ let actual;
1086
+ for (;;) {
1087
+ const tree = treeResultSchema.safeParse(await this.host.request("tree", { peek: true }, { sessionId: session.id }).catch(() => null));
1088
+ const element = tree.success
1089
+ ? tree.data.elements.find((el) => (e.id !== undefined ? el.testID === e.id : labelOfElement(el).toLowerCase() === (e.label ?? "").toLowerCase()))
1090
+ : undefined;
1091
+ if (element !== undefined) {
1092
+ actual = typeof element.checked === "boolean" ? (element.checked ? "on" : "off") : typeof element.value === "string" ? element.value : undefined;
1093
+ // A switch takes "true" and "false" too (it reads checked=true in the list).
1094
+ const wanted = typeof element.checked === "boolean" && (e.value === "true" || e.value === "false") ? (e.value === "true" ? "on" : "off") : e.value;
1095
+ if (actual === wanted)
1096
+ return null;
1097
+ }
1098
+ if (Date.now() >= until)
1099
+ return expectFailure(e, seconds, actual);
1100
+ await sleep(100);
1101
+ }
1102
+ }
1103
+ // -- inspect (Tapline 1.1): from the screen to the code
1104
+ async inspect(args) {
1105
+ const session = this.host.current();
1106
+ if (session === undefined)
1107
+ return this.notConnected();
1108
+ let raw;
1109
+ try {
1110
+ raw = await this.host.request("inspect", { target: toWireTarget(args.target) }, { sessionId: session.id });
1111
+ }
1112
+ catch (error) {
1113
+ return fail(describeError(error));
1114
+ }
1115
+ const parsed = inspectResultSchema.safeParse(raw);
1116
+ if (!parsed.success)
1117
+ return fail("Error internal: the app returned an inspect answer the host could not read.");
1118
+ return text(formatInspect(parsed.data));
1119
+ }
1120
+ // -- audit (AGENT-TOOLS 1.8 §10)
1121
+ async audit(args = {}) {
1122
+ const session = this.host.current();
1123
+ if (session === undefined)
1124
+ return this.notConnected();
1125
+ if (!session.capabilities.includes("audit")) {
1126
+ return fail("Error unsupported: this app's Tapline is too old for audit; update react-native-tapline (check_layout works meanwhile).");
1127
+ }
1128
+ let raw;
1129
+ try {
1130
+ raw = await this.host.request("audit", args.types !== undefined ? { types: args.types } : {}, { sessionId: session.id });
1131
+ }
1132
+ catch (error) {
1133
+ return fail(describeError(error));
1134
+ }
1135
+ const parsed = auditResultSchema.safeParse(raw);
1136
+ if (!parsed.success)
1137
+ return fail("Error internal: the app returned an audit the host could not read.");
1138
+ const title = auditTitle(await this.currentRoute(), session.app.name);
1139
+ return text([formatAudit(title, parsed.data, args.types), ...this.takePersonNote()].join("\n"));
1140
+ }
605
1141
  // -- check_layout
606
1142
  async checkLayout(_args = {}) {
607
1143
  const session = this.host.current();
@@ -754,32 +1290,42 @@ export class TaplineTools {
754
1290
  /** Runs a request, then (unless disabled) appends the fresh element list and how long the app took to settle. */
755
1291
  async act(method, params, describe, withLook, screenshot, extraNotes,
756
1292
  /** True when the result means nothing was done: the first line is then the whole answer. */
757
- nothingDone) {
1293
+ nothingDone,
1294
+ /** Runs inside the agent's turn, right before the request: an answer stops the call there (the policy). */
1295
+ preflight) {
758
1296
  const session = this.host.current();
759
1297
  if (session === undefined)
760
1298
  return this.notConnected();
761
- if (this.personActed) {
762
- const target = typeof params === "object" && params !== null ? params.target : undefined;
763
- if (typeof target === "object" && target !== null && "ref" in target) {
764
- // The error says it already, so the note isn't repeated.
765
- this.personNotePending = false;
766
- return fail("Error stale: a person used the app since your last look, so refs from that list may point elsewhere. Look again, or target by id, label or text.");
767
- }
768
- }
1299
+ const stale = this.staleRef(params);
1300
+ if (stale !== null)
1301
+ return stale;
769
1302
  let result;
1303
+ this.actionStartedAt = Date.now();
770
1304
  try {
771
1305
  const timeoutMs = answerTimeoutMs(method, params);
772
- const run = () => this.host.request(method, params, { sessionId: session.id, ...(timeoutMs !== undefined ? { timeoutMs } : {}) });
1306
+ const run = async () => {
1307
+ if (preflight !== undefined) {
1308
+ const stop = await preflight();
1309
+ if (stop !== null)
1310
+ throw new PolicyStop(stop);
1311
+ }
1312
+ return this.host.request(method, params, { sessionId: session.id, ...(timeoutMs !== undefined ? { timeoutMs } : {}) });
1313
+ };
773
1314
  // With the live viewer, a person watching goes first.
774
1315
  result = this.options.control ? await this.options.control.runAgent(run) : await run();
775
1316
  }
776
1317
  catch (error) {
1318
+ if (error instanceof PolicyStop)
1319
+ return error.answer;
1320
+ this.logExtra = undefined;
777
1321
  this.showAction("agent", method, undefined, params, error instanceof TaplineError ? error.code : "internal", error instanceof TaplineError ? reasonOf(error.code, error.message) : undefined);
778
1322
  return fail(describeError(error));
779
1323
  }
780
1324
  const line = describe(result);
781
- if (nothingDone?.(result) === true)
1325
+ if (nothingDone?.(result) === true) {
1326
+ this.logExtra = undefined;
782
1327
  return text([line, ...this.takePersonNote()].join("\n"));
1328
+ }
783
1329
  this.showAction("agent", method, result, params);
784
1330
  let response;
785
1331
  this.lastLookRoute = undefined;
@@ -826,8 +1372,14 @@ export class TaplineTools {
826
1372
  }
827
1373
  }
828
1374
  if (this.recording !== null && !this.replaying) {
829
- const { params: stable, fragile } = stableParams(params, result);
830
- const step = { method, params: stable };
1375
+ const { params: chosen, fragile } = stableParams(params, result);
1376
+ // A ref saved as a label or text that several elements share needs its index (AGENT-TOOLS 1.8 §9.4).
1377
+ const stable = await this.withIndexIfShared(params, chosen, result, session.id);
1378
+ // A password field's text is never saved in a script (Tapwright's review, M4).
1379
+ const secret = method === "type" && typeof result === "object" && result !== null && "secure" in result && result.secure === true;
1380
+ const step = { method, params: secret ? { ...stable, text: "" } : stable };
1381
+ if (secret)
1382
+ step.secret = true;
831
1383
  if (this.lastLookRoute !== undefined)
832
1384
  step.route = this.lastLookRoute;
833
1385
  if (fragile)
@@ -836,6 +1388,39 @@ export class TaplineTools {
836
1388
  }
837
1389
  return response;
838
1390
  }
1391
+ /**
1392
+ * When the agent's ref was saved as a testID, label or text that more than one element on screen has, add
1393
+ * the element's `index` among them (reading order), so the replay picks the same one instead of
1394
+ * answering `ambiguous`. One extra read, only while recording and only for ref targets.
1395
+ */
1396
+ async withIndexIfShared(original, chosen, result, sessionId) {
1397
+ const before = typeof original === "object" && original !== null ? original.target : undefined;
1398
+ const target = chosen["target"];
1399
+ if (typeof before !== "object" || before === null || !("ref" in before))
1400
+ return chosen;
1401
+ if (typeof target !== "object" || target === null || "ref" in target || "index" in target)
1402
+ return chosen;
1403
+ const element = typeof result === "object" && result !== null && "element" in result
1404
+ ? elementSchema.safeParse(result.element)
1405
+ : null;
1406
+ if (!element?.success)
1407
+ return chosen;
1408
+ const tree = treeResultSchema.safeParse(await this.host.request("tree", { peek: true }, { sessionId }).catch(() => null));
1409
+ if (!tree.success)
1410
+ return chosen;
1411
+ const t = target;
1412
+ const same = tree.data.elements.filter((el) => typeof t.testID === "string"
1413
+ ? el.testID === t.testID
1414
+ : typeof t.label === "string"
1415
+ ? labelOfElement(el).toLowerCase() === t.label.toLowerCase()
1416
+ : typeof t.text === "string"
1417
+ ? el.text === t.text
1418
+ : false);
1419
+ if (same.length < 2)
1420
+ return chosen;
1421
+ const at = same.findIndex((el) => el.ref === element.data.ref);
1422
+ return at < 0 ? chosen : { ...chosen, target: { ...target, index: at } };
1423
+ }
839
1424
  // -- the live viewer
840
1425
  /** An element's frame (points) as 0..1 of the screen. */
841
1426
  normalized(frame) {
@@ -859,15 +1444,43 @@ export class TaplineTools {
859
1444
  ...(point ? { point } : {}),
860
1445
  };
861
1446
  }
1447
+ /** One line in the action log (§12). Never fails the action. */
1448
+ logAction(by, tool, params, result, code, reason) {
1449
+ if (this.actionLog === null)
1450
+ return;
1451
+ const element = typeof result === "object" && result !== null && "element" in result
1452
+ ? elementSchema.safeParse(result.element)
1453
+ : null;
1454
+ const session = this.host.current();
1455
+ this.actionLog.write({
1456
+ by,
1457
+ tool,
1458
+ ...loggedTarget(params),
1459
+ // typed text never reaches the log, not even as the field's value (§12)
1460
+ ...(element?.success ? { element: formatElement({ ...element.data, ref: "", value: null, placeholder: null }, { actions: false }) } : {}),
1461
+ ok: code === undefined,
1462
+ ...(code !== undefined ? { code } : {}),
1463
+ ...(reason !== undefined ? { reason } : {}),
1464
+ ...(by === "agent" && this.actionStartedAt !== undefined ? { ms: Date.now() - this.actionStartedAt } : {}),
1465
+ ...(session !== undefined ? { app: session.app.id ?? session.app.name } : {}),
1466
+ ...(by === "agent" && this.batch !== undefined ? { batch: this.batch } : {}),
1467
+ ...(by === "agent" ? (this.logExtra ?? {}) : {}),
1468
+ });
1469
+ if (by === "agent")
1470
+ this.logExtra = undefined;
1471
+ }
862
1472
  showAction(by, method, result, params, code, reason) {
863
1473
  const onAction = this.options.onAction;
864
- if (onAction === undefined)
1474
+ if (onAction === undefined) {
1475
+ this.logAction(by, by === "agent" ? agentKind(method) : method, params, result, code, reason);
865
1476
  return;
1477
+ }
866
1478
  const target = typeof params === "object" && params !== null ? params.target : undefined;
867
1479
  const t = typeof target === "object" && target !== null ? target : undefined;
868
1480
  const screen = this.lastScreen;
869
1481
  const point = t && typeof t.x === "number" && typeof t.y === "number" && screen ? { x: t.x / screen.w, y: t.y / screen.h } : undefined;
870
1482
  const kind = by === "agent" ? agentKind(method) : method;
1483
+ this.logAction(by, kind, params, result, code, reason);
871
1484
  const described = this.describeAction(kind, result, point);
872
1485
  // A failed step has no element in its answer: name it from the target the agent gave, if it's words.
873
1486
  const named = described.label ??
@@ -968,7 +1581,10 @@ export class TaplineTools {
968
1581
  }
969
1582
  };
970
1583
  try {
971
- return this.options.control ? await this.options.control.runPerson(run) : await run();
1584
+ const outcome = this.options.control ? await this.options.control.runPerson(run) : await run();
1585
+ // The log (§12): a person's input too, its typed text only as a length.
1586
+ this.logAction("person", outcome.action?.kind ?? input.type, input.type === "text" ? { text: input.text } : {}, undefined, outcome.ok ? undefined : outcome.code, outcome.reason);
1587
+ return outcome;
972
1588
  }
973
1589
  catch (error) {
974
1590
  return { ok: false, code: "internal", message: error instanceof Error ? error.message : String(error) };
@@ -1015,11 +1631,28 @@ export class TaplineTools {
1015
1631
  const links = this.options.viewerLinks?.() ?? [];
1016
1632
  if (links.length === 0)
1017
1633
  return fail("Error unsupported: this host has no live viewer.");
1018
- const lines = [
1019
- "Live viewer (watch the app, see what you do, and let the user tap and type too):",
1020
- ...links.map((link) => `- ${link}`),
1021
- "The link carries a key: give it to the user only. While they use the viewer, their touches go first and your actions wait.",
1022
- ];
1634
+ const open = this.options.openViewer;
1635
+ const local = links[0];
1636
+ let lines;
1637
+ if (open !== undefined && this.options.viewerLinkToAgent !== true && local !== undefined) {
1638
+ // The key in the link can approve a held-back touch: it goes to the browser, never to the agent.
1639
+ const opened = open(local);
1640
+ lines = [
1641
+ opened
1642
+ ? "Opened the live view in a browser on this computer: the user watches the app and what you do there, and can tap and type too."
1643
+ : "Couldn't open a browser on this computer. Ask the user to start the MCP server with --viewer-link if they want the link.",
1644
+ "The link isn't shown to you: its key can approve touches that need a person's OK.",
1645
+ ...(links.length > 1 ? ["For their own devices on Tailscale, the user finds that link in this server's log."] : []),
1646
+ "While the user acts in the live view, their touches go first and your actions wait.",
1647
+ ];
1648
+ }
1649
+ else {
1650
+ lines = [
1651
+ "Live viewer (watch the app, see what you do, and let the user tap and type too):",
1652
+ ...links.map((link) => `- ${link}`),
1653
+ "The link carries a key: give it to the user only. While they use the viewer, their touches go first and your actions wait.",
1654
+ ];
1655
+ }
1023
1656
  const feed = this.options.viewerFeed?.();
1024
1657
  const session = this.host.current();
1025
1658
  if (feed?.on === true && session !== undefined) {
@@ -1064,66 +1697,99 @@ export class TaplineTools {
1064
1697
  else
1065
1698
  content.push({ type: "image", data: image.data, mimeType: image.mime });
1066
1699
  }
1067
- async compareScreen(args) {
1068
- const session = this.host.current();
1069
- if (session === undefined)
1070
- return this.notConnected();
1071
- const name = args.name.trim();
1072
- if (!SCRIPT_NAME.test(name))
1073
- return fail("Error bad_params: name the reference with letters, digits, '.', '_' and '-'.");
1700
+ /** Where this app's reference pictures are kept, or an error answer. */
1701
+ referenceFolder(session) {
1074
1702
  const dir = this.options.baselinesDir;
1075
1703
  if (dir === undefined)
1076
1704
  return fail("Error internal: no folder for reference pictures is configured.");
1077
- const appFolder = (session.app.id ?? session.app.name).replace(/[^\w.-]/g, "_");
1078
- const path = join(dir, appFolder, `${name}.jpg`);
1705
+ return join(dir, (session.app.id ?? session.app.name).replace(/[^\w.-]/g, "_"));
1706
+ }
1707
+ /** The element list for naming regions and masking, read as a peek: the agent's refs and baseline must stay as they are. */
1708
+ async peekElements(sessionId) {
1709
+ try {
1710
+ const tree = treeResultSchema.safeParse(await this.host.request("tree", { peek: true }, { sessionId }));
1711
+ if (tree.success && tree.data.screen)
1712
+ return { scaleBase: tree.data.screen.width, elements: tree.data.elements };
1713
+ }
1714
+ catch {
1715
+ // positions stay in pixels
1716
+ }
1717
+ return { scaleBase: undefined, elements: [] };
1718
+ }
1719
+ /**
1720
+ * Takes the screen and compares it with the saved reference called `name`, never saving. `referencesDirs` are the
1721
+ * folders to look in, in order. A missing reference hands back the screenshot, so the caller may save it.
1722
+ */
1723
+ async checkAgainstReference(session, name, options) {
1079
1724
  let shot;
1080
1725
  try {
1081
- shot = await this.takeShot(session.id, args.width ?? DEFAULT_SCREENSHOT_WIDTH, 0.9);
1726
+ shot = await this.takeShot(session.id, options.width, 0.9);
1082
1727
  }
1083
1728
  catch (error) {
1084
- return fail(describeError(error));
1729
+ return { kind: "error", line: describeError(error) };
1085
1730
  }
1086
- if (args.update === true || !existsSync(path)) {
1087
- try {
1088
- mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
1089
- writeFileSync(path, Buffer.from(shot.data, "base64"), { mode: 0o600 });
1090
- }
1091
- catch (error) {
1092
- return fail(`Error internal: couldn't save the reference: ${error instanceof Error ? error.message : String(error)}`);
1093
- }
1094
- return text(`Saved the reference picture "${name}" (${shot.width ?? "?"}x${shot.height ?? "?"}) in ${path}. Next time, compare_screen { name: "${name}" } shows what changed.`);
1095
- }
1096
- let diff;
1731
+ this.lastMaskElements = undefined;
1732
+ if (!SCRIPT_NAME.test(name))
1733
+ return { kind: "error", line: BAD_REFERENCE_NAME };
1734
+ const path = options.referencesDirs.map((dir) => join(dir, `${name}.jpg`)).find((candidate) => existsSync(candidate));
1735
+ if (path === undefined)
1736
+ return { kind: "missing", shot };
1737
+ let reference;
1097
1738
  try {
1098
- diff = diffImages({ mime: "image/jpeg", data: readFileSync(path).toString("base64") }, shot);
1739
+ reference = { mime: "image/jpeg", data: readFileSync(path).toString("base64") };
1099
1740
  }
1100
1741
  catch (error) {
1101
- return fail(`Error bad_params: ${error instanceof Error ? error.message : String(error)} To start over, use update: true.`);
1742
+ // A reference that can't be read (no permission, a folder in its place) is a failed check, never a crash.
1743
+ return { kind: "error", line: `Error bad_params: the reference "${name}" can't be read (${error instanceof Error ? error.message : String(error)}). To start over, use update: true.` };
1102
1744
  }
1103
- if (diff.changedPixels === 0)
1104
- return text(`No visible change from the reference "${name}".`);
1105
- // Name the regions in points, with the elements they touch, from a fresh element list.
1106
- let scale = 1;
1107
- let elements = [];
1745
+ let refSize;
1746
+ let shotSize;
1108
1747
  try {
1109
- // `peek`: the agent isn't given this list, so its refs and its baseline for changes must stay as they are.
1110
- const tree = treeResultSchema.safeParse(await this.host.request("tree", { peek: true }, { sessionId: session.id }));
1111
- if (tree.success && tree.data.screen) {
1112
- scale = diff.width / tree.data.screen.width;
1113
- elements = tree.data.elements;
1748
+ refSize = imageSize(reference);
1749
+ shotSize = imageSize(shot);
1750
+ }
1751
+ catch (error) {
1752
+ return { kind: "error", line: `Error bad_params: ${error instanceof Error ? error.message : String(error)} To start over, use update: true.` };
1753
+ }
1754
+ if (refSize.width !== shotSize.width || refSize.height !== shotSize.height) {
1755
+ return {
1756
+ kind: "size",
1757
+ message: `the reference "${name}" is ${refSize.width}x${refSize.height} px and this screen is ${shotSize.width}x${shotSize.height}; take it at the same width, or update: true.`,
1758
+ };
1759
+ }
1760
+ let seen;
1761
+ const notes = [];
1762
+ const mask = [];
1763
+ if (options.ignore.length > 0) {
1764
+ seen = await this.peekElements(session.id);
1765
+ this.lastMaskElements = seen.elements;
1766
+ if (seen.scaleBase === undefined) {
1767
+ return { kind: "error", line: "Error internal: couldn't read the screen's elements to paint out what ignore names." };
1114
1768
  }
1769
+ const masked = ignoreRects(options.ignore, seen.elements, shotSize.width / seen.scaleBase, shotSize);
1770
+ mask.push(...masked.rects);
1771
+ notes.push(...masked.notes);
1115
1772
  }
1116
- catch {
1117
- // positions stay in pixels
1773
+ let diff;
1774
+ try {
1775
+ diff = diffImages(reference, shot, { mask });
1776
+ }
1777
+ catch (error) {
1778
+ return { kind: "error", line: `Error bad_params: ${error instanceof Error ? error.message : String(error)} To start over, use update: true.` };
1118
1779
  }
1780
+ // The pass bar compares the exact share; the rounded percent is only for showing (a few pixels round to 0%).
1781
+ const exactPercent = (diff.changedPixels / Math.max(1, diff.width * diff.height)) * 100;
1782
+ if (diff.changedPixels === 0 || exactPercent <= options.threshold) {
1783
+ return { kind: "match", percent: diff.changedPercent, notes };
1784
+ }
1785
+ // Name the regions in points, with the elements they touch.
1786
+ const found = seen ?? (await this.peekElements(session.id));
1787
+ const scale = found.scaleBase === undefined ? 1 : diff.width / found.scaleBase;
1119
1788
  const unit = scale === 1 ? "pixels" : "points";
1120
- const lines = [
1121
- `${diff.changedPercent}% of the screen changed from the reference "${name}", in ${diff.boxes.length} region${diff.boxes.length === 1 ? "" : "s"} (${unit}):`,
1122
- ];
1123
- for (const box of diff.boxes) {
1789
+ const regions = diff.boxes.map((box) => {
1124
1790
  const b = { x: box.x / scale, y: box.y / scale, width: box.width / scale, height: box.height / scale };
1125
1791
  // The smallest named elements there: a page-sized container overlaps every change.
1126
- const near = elements
1792
+ const near = found.elements
1127
1793
  .filter((e) => e.frame && e.frame.x < b.x + b.width && e.frame.x + e.frame.width > b.x && e.frame.y < b.y + b.height && e.frame.y + e.frame.height > b.y)
1128
1794
  .filter((e) => (e.text ?? "") !== "" || (e.label ?? "") !== "" || (e.testID ?? "") !== "")
1129
1795
  .map((e) => ({ e, area: (e.frame?.width ?? 0) * (e.frame?.height ?? 0) }))
@@ -1131,12 +1797,289 @@ export class TaplineTools {
1131
1797
  .sort((x, y) => x.area - y.area)
1132
1798
  .slice(0, 2)
1133
1799
  .map(({ e }) => describeElement(e));
1134
- lines.push(`- @${Math.round(b.x)},${Math.round(b.y)} ${Math.round(b.width)}x${Math.round(b.height)}${near.length > 0 ? `: ${near.join(", ")}` : ""}`);
1800
+ return { at: `@${Math.round(b.x)},${Math.round(b.y)} ${Math.round(b.width)}x${Math.round(b.height)}`, near };
1801
+ });
1802
+ const count = regions.length;
1803
+ const where = `in ${count} region${count === 1 ? "" : "s"} (${unit})`;
1804
+ const lines = [`Differs from the reference "${name}": ${diff.changedPercent}% changed, ${where}:`];
1805
+ for (const region of regions.slice(0, MAX_REGION_LINES)) {
1806
+ lines.push(`- ${region.at}${region.near.length > 0 ? `: ${region.near.join(", ")}` : ""}`);
1807
+ }
1808
+ if (count > MAX_REGION_LINES)
1809
+ lines.push(`(+${count - MAX_REGION_LINES} more regions)`);
1810
+ const listed = regions.slice(0, MAX_REGION_LINES).map((region) => region.at).join(", ");
1811
+ const reason = `${diff.changedPercent}% changed from the reference "${name}" (threshold ${options.threshold}%), ${where}: ${listed}${count > MAX_REGION_LINES ? `, +${count - MAX_REGION_LINES} more` : ""}.`;
1812
+ return { kind: "differs", percent: diff.changedPercent, lines, reason, image: diff.image, notes };
1813
+ }
1814
+ /** Why a check does not hold (for `run`, as a step or as `matches`), or null when it matches. */
1815
+ referenceReason(check, name) {
1816
+ switch (check.kind) {
1817
+ case "match":
1818
+ return null;
1819
+ case "missing":
1820
+ return { code: "expect_failed", message: `no reference "${name}" yet; save one with compare_screen { name: "${name}" }.` };
1821
+ case "size":
1822
+ return { code: "bad_params", message: check.message };
1823
+ case "error": {
1824
+ const { code, message } = parseError(check.line);
1825
+ return { code, message };
1826
+ }
1827
+ case "differs":
1828
+ return { code: "expect_failed", message: check.reason };
1829
+ }
1830
+ }
1831
+ /** Whether the screen matches the reference `name` now (`expect.matches`): null when it does, else the reason. */
1832
+ async matchMissed(name, threshold, ignore) {
1833
+ this.lastMaskElements = undefined;
1834
+ if (!SCRIPT_NAME.test(name))
1835
+ return parseError(BAD_REFERENCE_NAME).message;
1836
+ const session = this.host.current();
1837
+ if (session === undefined)
1838
+ return "the app isn't connected";
1839
+ const folder = this.referenceFolder(session);
1840
+ if (typeof folder !== "string")
1841
+ return parseError(folder.content.map((block) => (block.type === "text" ? block.text : "")).join("\n")).message;
1842
+ if (this.replayContext?.update === true) {
1843
+ // --update-references: the check saves the screen into the script's folder, and holds.
1844
+ const saved = await this.saveReferenceHere(session, name, DEFAULT_SCREENSHOT_WIDTH, this.replayContext.referencesDir);
1845
+ return saved.isError === true ? parseError(saved.content.map((block) => (block.type === "text" ? block.text : "")).join("\n")).message : null;
1846
+ }
1847
+ const check = await this.checkAgainstReference(session, name, {
1848
+ threshold,
1849
+ ignore,
1850
+ width: DEFAULT_SCREENSHOT_WIDTH,
1851
+ referencesDirs: this.referencesDirs(folder),
1852
+ });
1853
+ this.keepDiff(check, name);
1854
+ return this.referenceReason(check, name)?.message ?? null;
1855
+ }
1856
+ /** Where a check looks for `<name>.jpg`: the folder next to a replayed script first, then the app's own store. */
1857
+ referencesDirs(folder) {
1858
+ return this.replayContext !== null ? [this.replayContext.referencesDir, folder] : [folder];
1859
+ }
1860
+ /** In a replay, a failed check leaves its red picture at `<script>.diff/<name>.jpg`. */
1861
+ keepDiff(check, name) {
1862
+ if (this.replayContext === null || check.kind !== "differs" || !SCRIPT_NAME.test(name))
1863
+ return;
1864
+ const path = join(this.replayContext.diffDir, `${name}.jpg`);
1865
+ try {
1866
+ mkdirSync(dirname(path), { recursive: true });
1867
+ writeFileSync(path, Buffer.from(check.image.data, "base64"));
1868
+ this.diffSaved = path;
1869
+ }
1870
+ catch {
1871
+ // the failure still reads without the picture
1872
+ }
1873
+ }
1874
+ /** Takes the screen and saves it as the reference `name` in `dir`. */
1875
+ async saveReferenceHere(session, name, width, dir) {
1876
+ try {
1877
+ return this.writeReference(dir, name, await this.takeShot(session.id, width, 0.9));
1878
+ }
1879
+ catch (error) {
1880
+ return fail(describeError(error));
1881
+ }
1882
+ }
1883
+ writeReference(folder, name, shot) {
1884
+ // Every write checks the name itself, whatever the caller checked: no way out of the references folder.
1885
+ if (!SCRIPT_NAME.test(name))
1886
+ return fail(BAD_REFERENCE_NAME);
1887
+ const path = join(folder, `${name}.jpg`);
1888
+ try {
1889
+ mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
1890
+ writeFileSync(path, Buffer.from(shot.data, "base64"), { mode: 0o600 });
1891
+ }
1892
+ catch (error) {
1893
+ return fail(`Error internal: couldn't save the reference: ${error instanceof Error ? error.message : String(error)}`);
1894
+ }
1895
+ let size = `${shot.width ?? "?"}x${shot.height ?? "?"}`;
1896
+ try {
1897
+ const real = imageSize(shot);
1898
+ size = `${real.width}x${real.height}`;
1899
+ }
1900
+ catch {
1901
+ // the app's own size stays
1902
+ }
1903
+ return text(`Saved the reference "${name}" (${size} px).`);
1904
+ }
1905
+ /** A `compare_screen` step of `run`: it never saves; it holds on a match and stops the batch otherwise. */
1906
+ async compareStep(args) {
1907
+ this.lastMaskElements = undefined;
1908
+ const session = this.host.current();
1909
+ if (session === undefined)
1910
+ return this.notConnected();
1911
+ const name = args.name.trim();
1912
+ if (!SCRIPT_NAME.test(name))
1913
+ return fail("Error bad_params: name the reference with letters, digits, '.', '_' and '-'.");
1914
+ const folder = this.referenceFolder(session);
1915
+ if (typeof folder !== "string")
1916
+ return folder;
1917
+ if (this.replayContext?.update === true) {
1918
+ return this.saveReferenceHere(session, name, args.width ?? DEFAULT_SCREENSHOT_WIDTH, this.replayContext.referencesDir);
1919
+ }
1920
+ const check = await this.checkAgainstReference(session, name, {
1921
+ threshold: args.threshold ?? DEFAULT_MATCH_THRESHOLD,
1922
+ ignore: args.ignore ?? [],
1923
+ width: args.width ?? DEFAULT_SCREENSHOT_WIDTH,
1924
+ referencesDirs: this.referencesDirs(folder),
1925
+ });
1926
+ this.keepDiff(check, name);
1927
+ const why = this.referenceReason(check, name);
1928
+ if (why !== null)
1929
+ return fail(`Error ${why.code}: ${why.message}`);
1930
+ if (check.kind !== "match")
1931
+ return fail("Error internal: the comparison gave no answer.");
1932
+ // A check that held inside `run` is kept in a recording too, with its options.
1933
+ const recorded = await this.recordCompare({
1934
+ name,
1935
+ ...(args.threshold !== undefined ? { threshold: args.threshold } : {}),
1936
+ ...(args.ignore !== undefined ? { ignore: args.ignore } : {}),
1937
+ ...(args.width !== undefined ? { width: args.width } : {}),
1938
+ });
1939
+ if (recorded !== null)
1940
+ return fail(recorded);
1941
+ return text([`Matches the reference "${name}": ${check.percent}% changed.`, ...check.notes.map((n) => `Note: ${n}`)].join("\n"));
1942
+ }
1943
+ /**
1944
+ * Saves a visual check that held as a `compareScreen` step of the recording, when one is running: the name, and the
1945
+ * threshold, the ignore list (refs made stable) and the width when given. `update` is never written. Answers an
1946
+ * error line when an ignore ref can't be kept in a script, else null (also when nothing is recording).
1947
+ */
1948
+ async recordCompare(check) {
1949
+ if (this.recording === null || this.replaying)
1950
+ return null;
1951
+ const session = this.host.current();
1952
+ let kept = check.ignore;
1953
+ if (session !== undefined && kept !== undefined && kept.length > 0) {
1954
+ const stable = await this.stableIgnore(session.id, kept, this.lastMaskElements);
1955
+ if (typeof stable === "string")
1956
+ return stable;
1957
+ kept = stable;
1958
+ }
1959
+ if (this.recording === null)
1960
+ return null;
1961
+ this.recording.steps.push({
1962
+ method: "compareScreen",
1963
+ params: {
1964
+ name: check.name.trim(),
1965
+ ...(check.threshold !== undefined ? { threshold: check.threshold } : {}),
1966
+ ...(kept !== undefined && kept.length > 0 ? { ignore: kept } : {}),
1967
+ ...(check.width !== undefined ? { width: check.width } : {}),
1968
+ },
1969
+ });
1970
+ return null;
1971
+ }
1972
+ /**
1973
+ * `ignore` as a script keeps it (AGENT-TOOLS 1.9 §14.2): a ref becomes the element's id, else label, else text,
1974
+ * read from the list now. A string is the error answer when a ref can't be kept.
1975
+ */
1976
+ async stableIgnore(sessionId, ignore, known) {
1977
+ if (!ignore.some((item) => !("width" in item) && item.ref !== undefined))
1978
+ return ignore;
1979
+ // The elements the comparison painted out when it had them, so the script keeps exactly that mask.
1980
+ const seen = known !== undefined ? { elements: known } : await this.peekElements(sessionId);
1981
+ const kept = [];
1982
+ for (const item of ignore) {
1983
+ if ("width" in item || item.ref === undefined) {
1984
+ kept.push(item);
1985
+ continue;
1986
+ }
1987
+ const element = seen.elements.find((e) => e.ref === item.ref);
1988
+ if (element === undefined)
1989
+ return `Error bad_params: ignore ${item.ref} isn't on the screen, so it can't be saved in a script.`;
1990
+ // When several elements share the name, the saved item keeps which one it was (`index`, counted as ignoreRects
1991
+ // counts: elements with a frame, in list order), so a replay paints out that one and not all of them.
1992
+ const framed = seen.elements.filter((e) => e.frame !== undefined);
1993
+ const withIndex = (saved, same) => {
1994
+ const matching = framed.filter(same);
1995
+ const at = matching.indexOf(element);
1996
+ return matching.length > 1 && at >= 0 ? { ...saved, index: at } : saved;
1997
+ };
1998
+ if (typeof element.testID === "string" && element.testID !== "") {
1999
+ const id = element.testID;
2000
+ kept.push(withIndex({ id }, (e) => e.testID === id));
2001
+ }
2002
+ else if (typeof element.label === "string" && element.label !== "") {
2003
+ const label = element.label.toLowerCase();
2004
+ kept.push(withIndex({ label: element.label }, (e) => labelOfElement(e).toLowerCase() === label));
2005
+ }
2006
+ else if (typeof element.text === "string" && element.text !== "") {
2007
+ const text = element.text.toLowerCase();
2008
+ kept.push(withIndex({ text: element.text }, (e) => (e.text ?? "").toLowerCase() === text));
2009
+ }
2010
+ else {
2011
+ return `Error bad_params: ignore ${describeElement(element)} has no id, label or text, so a script couldn't find it again; give it a testID, or ignore a rect {x, y, width, height}.`;
2012
+ }
2013
+ }
2014
+ return kept;
2015
+ }
2016
+ async compareScreen(args) {
2017
+ // Each check starts with no mask list: an update, or a first save, never records from an earlier comparison.
2018
+ this.lastMaskElements = undefined;
2019
+ const recording = this.recording !== null && !this.replaying;
2020
+ const session = this.host.current();
2021
+ // An ignore ref that a script couldn't find again is refused before anything is compared.
2022
+ if (recording && session !== undefined && args.ignore !== undefined && args.ignore.length > 0) {
2023
+ const stable = await this.stableIgnore(session.id, args.ignore);
2024
+ if (typeof stable === "string")
2025
+ return fail(stable);
2026
+ }
2027
+ const answer = await this.compareScreenNow(args);
2028
+ if (recording && answer.isError !== true) {
2029
+ // `update` is never written: the script checks against the reference it carries.
2030
+ const recorded = await this.recordCompare({
2031
+ name: args.name,
2032
+ ...(args.threshold !== undefined ? { threshold: args.threshold } : {}),
2033
+ ...(args.ignore !== undefined ? { ignore: args.ignore } : {}),
2034
+ ...(args.width !== undefined ? { width: args.width } : {}),
2035
+ });
2036
+ if (recorded !== null)
2037
+ return fail(recorded);
2038
+ }
2039
+ return answer;
2040
+ }
2041
+ async compareScreenNow(args) {
2042
+ const session = this.host.current();
2043
+ if (session === undefined)
2044
+ return this.notConnected();
2045
+ const name = args.name.trim();
2046
+ if (!SCRIPT_NAME.test(name))
2047
+ return fail("Error bad_params: name the reference with letters, digits, '.', '_' and '-'.");
2048
+ const folder = this.referenceFolder(session);
2049
+ if (typeof folder !== "string")
2050
+ return folder;
2051
+ const width = args.width ?? DEFAULT_SCREENSHOT_WIDTH;
2052
+ const save = (shot) => this.writeReference(folder, name, shot);
2053
+ if (args.update === true) {
2054
+ try {
2055
+ return save(await this.takeShot(session.id, width, 0.9));
2056
+ }
2057
+ catch (error) {
2058
+ return fail(describeError(error));
2059
+ }
2060
+ }
2061
+ const check = await this.checkAgainstReference(session, name, {
2062
+ threshold: args.threshold ?? DEFAULT_MATCH_THRESHOLD,
2063
+ ignore: args.ignore ?? [],
2064
+ width,
2065
+ referencesDirs: [folder],
2066
+ });
2067
+ switch (check.kind) {
2068
+ case "missing":
2069
+ return save(check.shot);
2070
+ case "error":
2071
+ return fail(check.line);
2072
+ case "size":
2073
+ return fail(`Error bad_params: ${check.message}`);
2074
+ case "match":
2075
+ return text([`Matches the reference "${name}": ${check.percent}% changed.`, ...check.notes.map((n) => `Note: ${n}`)].join("\n"));
2076
+ case "differs": {
2077
+ const lines = [...check.lines, "The image shows the screen now, faded, with the changed pixels in red.", ...check.notes.map((n) => `Note: ${n}`)];
2078
+ const content = [];
2079
+ this.deliver(session.id, check.image, lines, content);
2080
+ return { content: [{ type: "text", text: lines.join("\n") }, ...content] };
2081
+ }
1135
2082
  }
1136
- lines.push("The image shows the screen now, faded, with the changed pixels in red.");
1137
- const content = [];
1138
- this.deliver(session.id, diff.image, lines, content);
1139
- return { content: [{ type: "text", text: lines.join("\n") }, ...content] };
1140
2083
  }
1141
2084
  async filmstrip(args) {
1142
2085
  const session = this.host.current();
@@ -1151,8 +2094,13 @@ export class TaplineTools {
1151
2094
  const started = Date.now();
1152
2095
  if (args.tap !== undefined) {
1153
2096
  // A screenshot takes a few hundred ms on a phone, so frame 1 is the "before".
2097
+ // The tap is a touch like any other: the safety policy decides first.
2098
+ const pressParams = { target: toWireTarget(args.tap) };
2099
+ const gate = await this.policyGate(session, "tap", "press", pressParams, listsFor(this.policy, session.app));
2100
+ if ("stop" in gate)
2101
+ return gate.stop;
1154
2102
  frames.push(await this.takeShot(session.id, frameWidth, 0.8));
1155
- await this.host.request("press", { target: toWireTarget(args.tap) }, { sessionId: session.id });
2103
+ await this.host.request("press", pressParams, { sessionId: session.id });
1156
2104
  }
1157
2105
  for (let i = frames.length; i < count; i++) {
1158
2106
  const at = Date.now();
@@ -1178,9 +2126,10 @@ export class TaplineTools {
1178
2126
  return dir === undefined || !SCRIPT_NAME.test(name) ? null : join(dir, `${name}.json`);
1179
2127
  }
1180
2128
  async record(args) {
1181
- switch (args.action) {
2129
+ const op = args.action ?? (args.start === true ? "start" : args.stop === true ? "stop" : "status");
2130
+ switch (op) {
1182
2131
  case "start": {
1183
- const name = (args.name ?? "").trim();
2132
+ const name = (args.name ?? "").trim() || `session-${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, "")}`;
1184
2133
  if (this.scriptPath(name) === null) {
1185
2134
  return fail("Error bad_params: give the script a name of letters, digits, '.', '_' and '-' (e.g. \"checkout\").");
1186
2135
  }
@@ -1190,7 +2139,7 @@ export class TaplineTools {
1190
2139
  if (session === undefined)
1191
2140
  return this.notConnected();
1192
2141
  this.recording = { name, app: { name: session.app.name, ...(session.app.id ? { id: session.app.id } : {}) }, steps: [] };
1193
- return text(`Recording "${name}" on ${session.app.name}. Every action from now on is a step; wait_for steps check the app is where it should be. Prefer testIDs: they replay best. Stop with record { action: "stop" }.`);
2142
+ return text(`Recording "${name}" on ${session.app.name}. Every action from now on is a step; wait_for steps check the app is where it should be. Prefer testIDs: they replay best. Stop with record { stop: true }.`);
1194
2143
  }
1195
2144
  case "status":
1196
2145
  return text(this.recording === null
@@ -1204,16 +2153,17 @@ export class TaplineTools {
1204
2153
  case "stop": {
1205
2154
  const recording = this.recording;
1206
2155
  if (recording === null)
1207
- return fail("Error bad_params: not recording; start with record { action: \"start\", name }.");
1208
- const path = this.scriptPath(recording.name);
2156
+ return fail("Error bad_params: not recording; start with record { start: true, name }.");
2157
+ const path = args.path !== undefined && args.path.trim() !== "" ? args.path.trim() : this.scriptPath(recording.name);
1209
2158
  if (path === null)
1210
- return fail("Error internal: no scripts folder is configured.");
2159
+ return fail("Error internal: no scripts folder is configured; give a path.");
1211
2160
  const script = {
2161
+ format: "agent-tools-script",
1212
2162
  version: 1,
1213
2163
  name: recording.name,
1214
- app: recording.app,
1215
- recordedAt: new Date().toISOString(),
1216
- steps: recording.steps,
2164
+ app: recording.app.id ?? recording.app.name,
2165
+ fresh: true,
2166
+ steps: scriptSteps(recording.steps),
1217
2167
  };
1218
2168
  try {
1219
2169
  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
@@ -1224,20 +2174,60 @@ export class TaplineTools {
1224
2174
  }
1225
2175
  this.recording = null;
1226
2176
  const count = script.steps.length;
1227
- const lines = [`Saved "${script.name}": ${count} step${count === 1 ? "" : "s"} in ${path}. Replay it with replay { name: "${script.name}" }.`];
1228
- script.steps.forEach((step, i) => {
1229
- if (step.fragile) {
1230
- const target = step.params["target"];
1231
- lines.push(`Note: step ${i + 1} targets ref ${String(target.ref)}, which won't exist on a new run. Give that element a testID, or record it again by its text or label.`);
1232
- }
1233
- });
1234
- if (script.steps.some((step) => step.method === "type")) {
1235
- lines.push("The script keeps the text typed in it (test passwords too): keep it private.");
2177
+ // AGENT-TOOLS 1.9 §14.3: the references the steps use travel with the script.
2178
+ const copied = this.copyReferences(script.steps, recording.app, path);
2179
+ const withPictures = copied === null ? "" : `, and ${copied.count} reference picture${copied.count === 1 ? "" : "s"} to ${copied.folder}`;
2180
+ const lines = [`Saved ${count} step${count === 1 ? "" : "s"} to ${path}${withPictures}.`];
2181
+ const ref = refStep(script.steps);
2182
+ if (ref !== null) {
2183
+ lines.push(`Note: step ${ref + 1} targets a ref, which won't exist on a new run, so replay refuses the script. Give that element a testID, or record it again by its text or label.`);
2184
+ }
2185
+ const secrets = recording.steps.filter((step) => step.secret === true).length;
2186
+ if (secrets > 0) {
2187
+ lines.push(`${secrets} step${secrets === 1 ? " types" : "s type"} into a password field: that text isn't saved. Put a test password into the script before you replay it.`);
2188
+ }
2189
+ if (recording.steps.some((step) => step.method === "type" && step.secret !== true)) {
2190
+ lines.push("The script keeps the other text typed in it: keep it private.");
1236
2191
  }
1237
2192
  return text(lines.join("\n"));
1238
2193
  }
1239
2194
  }
1240
2195
  }
2196
+ /** `<script dir>/<script name>.references` and `.diff` for a script file. */
2197
+ scriptSideFolder(path, kind) {
2198
+ return join(dirname(path), `${basename(path).replace(/\.json$/i, "")}.${kind}`);
2199
+ }
2200
+ /** Copies each reference the steps use from the app's store next to the script; null when there are none. */
2201
+ copyReferences(steps, app, scriptPath) {
2202
+ const dir = this.options.baselinesDir;
2203
+ if (dir === undefined)
2204
+ return null;
2205
+ const names = new Set();
2206
+ for (const step of steps) {
2207
+ const p = step;
2208
+ if (step.tool === "compare_screen" && typeof p["name"] === "string" && SCRIPT_NAME.test(p["name"]))
2209
+ names.add(p["name"]);
2210
+ if (step.expect?.matches !== undefined && SCRIPT_NAME.test(step.expect.matches))
2211
+ names.add(step.expect.matches);
2212
+ }
2213
+ const store = join(dir, (app.id ?? app.name).replace(/[^\w.-]/g, "_"));
2214
+ const folder = this.scriptSideFolder(scriptPath, "references");
2215
+ let count = 0;
2216
+ for (const name of names) {
2217
+ const from = join(store, `${name}.jpg`);
2218
+ if (!existsSync(from))
2219
+ continue;
2220
+ try {
2221
+ mkdirSync(folder, { recursive: true });
2222
+ copyFileSync(from, join(folder, `${name}.jpg`));
2223
+ count++;
2224
+ }
2225
+ catch {
2226
+ // the script still saved; the picture stays in the store
2227
+ }
2228
+ }
2229
+ return count === 0 ? null : { count, folder };
2230
+ }
1241
2231
  scriptList() {
1242
2232
  const dir = this.options.scriptsDir;
1243
2233
  if (dir === undefined || !existsSync(dir))
@@ -1245,11 +2235,10 @@ export class TaplineTools {
1245
2235
  const lines = [];
1246
2236
  for (const file of readdirSync(dir).filter((f) => f.endsWith(".json")).sort()) {
1247
2237
  try {
1248
- const parsed = scriptSchema.safeParse(JSON.parse(readFileSync(join(dir, file), "utf8")));
1249
- if (!parsed.success)
2238
+ const s = readScript(readFileSync(join(dir, file), "utf8"));
2239
+ if (s === null)
1250
2240
  continue;
1251
- const s = parsed.data;
1252
- lines.push(`- ${s.name} (${s.steps.length} step${s.steps.length === 1 ? "" : "s"}, recorded ${s.recordedAt.slice(0, 10)} on ${s.app.name})`);
2241
+ lines.push(`- ${s.name} (${s.steps.length} step${s.steps.length === 1 ? "" : "s"}${s.app ? `, ${s.app}` : ""})`);
1253
2242
  }
1254
2243
  catch {
1255
2244
  // skip unreadable files
@@ -1257,66 +2246,68 @@ export class TaplineTools {
1257
2246
  }
1258
2247
  return lines.length === 0 ? "No saved scripts." : ["Saved scripts:", ...lines].join("\n");
1259
2248
  }
1260
- async replay(args) {
2249
+ async replay(args, options = {}) {
1261
2250
  const name = (args.name ?? "").trim();
1262
- if (name === "")
2251
+ const given = (args.path ?? "").trim();
2252
+ if (name === "" && given === "")
1263
2253
  return text(this.scriptList());
1264
2254
  if (this.recording !== null)
1265
2255
  return fail(`Error bad_params: stop recording "${this.recording.name}" before replaying.`);
1266
- const path = this.scriptPath(name);
2256
+ const path = given !== "" ? given : this.scriptPath(name);
1267
2257
  if (path === null || !existsSync(path))
1268
- return fail(`Error not_found: no script "${name}".\n${this.scriptList()}`);
2258
+ return fail(`Error not_found: no script "${given || name}".\n${this.scriptList()}`);
1269
2259
  let script;
1270
2260
  try {
1271
- const parsed = scriptSchema.safeParse(JSON.parse(readFileSync(path, "utf8")));
1272
- if (!parsed.success)
1273
- return fail(`Error bad_params: ${path} isn't a Tapline script.`);
1274
- script = parsed.data;
2261
+ script = readScript(readFileSync(path, "utf8"));
1275
2262
  }
1276
2263
  catch (error) {
1277
2264
  return fail(`Error internal: couldn't read ${path}: ${error instanceof Error ? error.message : String(error)}`);
1278
2265
  }
2266
+ if (script === null)
2267
+ return fail(`Error bad_params: ${path} isn't a script (format agent-tools-script, version 1).`);
2268
+ const ref = refStep(script.steps);
2269
+ if (ref !== null)
2270
+ return fail(`Error bad_params: step ${ref + 1} targets a ref; a script needs an id, label, text or point.`);
2271
+ if (script.steps.length === 0)
2272
+ return text(`Ran 0 of 0 steps in 0.0 s.`);
1279
2273
  const session = this.host.current();
1280
2274
  if (session === undefined)
1281
2275
  return this.notConnected();
1282
- const started = Date.now();
1283
- const total = script.steps.length;
1284
- const lines = [];
1285
- const seconds = (ms) => (Math.round(ms / 100) / 10).toFixed(1);
2276
+ const notes = [];
2277
+ script.steps.forEach((step, i) => {
2278
+ const target = step["target"];
2279
+ if (typeof target === "object" && target !== null && "x" in target) {
2280
+ notes.push(`Note (step ${i + 1}): a point target; it breaks when the layout changes.`);
2281
+ }
2282
+ });
2283
+ if (script.fresh !== false) {
2284
+ // Tapline can't close and open the app: it starts its JavaScript again (a development build).
2285
+ const fresh = await this.reload({});
2286
+ const said = fresh.content.map((block) => (block.type === "text" ? block.text : "")).join("\n");
2287
+ if (fresh.isError === true)
2288
+ notes.push("Note: couldn't start the app fresh (a release build has no reload); the script ran on the screen as it was.");
2289
+ else if (said.startsWith("Reload requested, but"))
2290
+ notes.push("Note: the app hadn't reconnected after starting fresh when the steps began.");
2291
+ }
1286
2292
  this.replaying = true;
2293
+ this.replayContext = {
2294
+ referencesDir: this.scriptSideFolder(path, "references"),
2295
+ diffDir: this.scriptSideFolder(path, "diff"),
2296
+ update: options.updateReferences === true,
2297
+ };
1287
2298
  try {
1288
- for (const [i, step] of script.steps.entries()) {
1289
- const at = Date.now();
1290
- const label = `${i + 1} ${describeStep(step)}`;
1291
- if (step.method === "sleep") {
1292
- const s = typeof step.params["seconds"] === "number" ? step.params["seconds"] : 0;
1293
- await sleep(Math.min(30, Math.max(0, s)) * 1000);
1294
- lines.push(`${i + 1} ✓ ${describeStep(step)} (${seconds(Date.now() - at)} s)`);
1295
- continue;
1296
- }
1297
- const result = await this.act(step.method, step.params, () => "", true);
1298
- const problem = result.isError
1299
- ? result.content.map((block) => (block.type === "text" ? block.text : "")).join("\n").trim()
1300
- : step.route !== undefined &&
1301
- this.lastLookRoute !== undefined &&
1302
- step.route.join(" › ") !== this.lastLookRoute.join(" › ")
1303
- ? `It ended on ${this.lastLookRoute.join(" › ")}; the recording ended on ${step.route.join(" › ")}.`
1304
- : null;
1305
- if (problem !== null) {
1306
- return fail([
1307
- `Replay "${script.name}" failed at step ${i + 1} of ${total}: ${describeStep(step)}`,
1308
- problem,
1309
- ...(lines.length > 0 ? ["Steps before it passed:", ...lines] : []),
1310
- ...(step.fragile ? ["Note: this step targets a ref from the recording run; give the element a testID."] : []),
1311
- ].join("\n"));
1312
- }
1313
- lines.push(`${label.replace(/^(\d+) /, "$1 ✓ ")} (${seconds(Date.now() - at)} s)`);
1314
- }
2299
+ const answer = await this.run({ steps: script.steps, look: args.look, screenshot: args.screenshot });
2300
+ if (notes.length === 0)
2301
+ return answer;
2302
+ const [top, ...rest] = answer.content;
2303
+ return top !== undefined && top.type === "text"
2304
+ ? { ...answer, content: [{ type: "text", text: [top.text.trimEnd(), ...notes].join("\n") }, ...rest] }
2305
+ : answer;
1315
2306
  }
1316
2307
  finally {
1317
2308
  this.replaying = false;
2309
+ this.replayContext = null;
1318
2310
  }
1319
- return text([`Replayed "${script.name}" on ${session.app.name}: ${total} of ${total} steps passed in ${seconds(Date.now() - started)} s.`, ...lines].join("\n"));
1320
2311
  }
1321
2312
  async withLook(session, line, screenshot = false) {
1322
2313
  // The session may have been replaced (app restarted) in the meantime.
@@ -1329,21 +2320,275 @@ export class TaplineTools {
1329
2320
  const head = first !== undefined && first.type === "text" ? first.text : "";
1330
2321
  return { content: [{ type: "text", text: `${line}\n${head}` }, ...rest] };
1331
2322
  }
1332
- /** The line (without its actions) of the element a result carries under `key`. */
1333
- elementName(result, key = "element") {
2323
+ /**
2324
+ * The element a result carries under `key`, for an answer's first line: without its actions, position or
2325
+ * size (AGENT-TOOLS 1.8 §13 rule 1; the change block or list has them). `withoutValue` for type_text,
2326
+ * whose sentence says the value (rule 5).
2327
+ */
2328
+ elementName(result, key = "element", withoutValue = false) {
1334
2329
  if (typeof result === "object" && result !== null && key in result) {
1335
2330
  const element = elementSchema.safeParse(result[key]);
1336
- if (element.success)
1337
- return formatElement(element.data, { actions: false });
2331
+ if (element.success) {
2332
+ const shown = withoutValue ? { ...element.data, value: null, placeholder: null } : element.data;
2333
+ return formatElement(shown, { actions: false, frame: false });
2334
+ }
1338
2335
  }
1339
2336
  return "element";
1340
2337
  }
2338
+ // -- the safety policy (AGENT-TOOLS 1.8 §11)
2339
+ /**
2340
+ * A guarded touch: the app resolves the target in a dry run, the policy's words are matched against the
2341
+ * element's own label and text, and a confirm word waits for a person's OK in the live view. The check runs
2342
+ * inside the agent's turn, right before the touch, so nothing a person does in between can change what is
2343
+ * touched; when the check can't be made, nothing is touched (fail closed).
2344
+ */
2345
+ async guarded(tool, method, params, dryRun, perform) {
2346
+ const session = this.host.current();
2347
+ if (session === undefined)
2348
+ return this.notConnected();
2349
+ this.logExtra = undefined;
2350
+ this.actionStartedAt = Date.now();
2351
+ const lists = listsFor(this.policy, session.app);
2352
+ if (!dryRun && lists.confirm.length === 0 && lists.block.length === 0)
2353
+ return perform();
2354
+ const stale = this.staleRef(params);
2355
+ if (stale !== null)
2356
+ return stale;
2357
+ if (dryRun) {
2358
+ const resolved = await this.resolveForPolicy(session, method, params);
2359
+ this.logExtra = { dry_run: true };
2360
+ if (resolved.kind === "error") {
2361
+ this.showAction("agent", method, undefined, params, resolved.code, resolved.reason);
2362
+ return fail(resolved.text);
2363
+ }
2364
+ const verdict = this.policyWords(lists, resolved.elements);
2365
+ const shown = resolved.elements[0];
2366
+ if (verdict.kind === "blocked") {
2367
+ this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "blocked");
2368
+ return fail(this.blockedLine(verdict.element, verdict.word));
2369
+ }
2370
+ const would = this.wouldLine(tool, shown, resolved.point, resolved.value);
2371
+ const needs = verdict.kind === "confirm" ? ` It needs a person's OK (policy word ${JSON.stringify(verdict.word)}).` : "";
2372
+ this.showAction("agent", method, shown !== undefined ? { element: shown } : undefined, params);
2373
+ return text([`${would}${needs}`, ...this.takePersonNote()].join("\n"));
2374
+ }
2375
+ let granted = false;
2376
+ const answer = await perform(async () => {
2377
+ const gate = await this.policyGate(session, tool, method, params, lists);
2378
+ if ("stop" in gate)
2379
+ return gate.stop;
2380
+ granted = gate.granted;
2381
+ if (granted)
2382
+ this.logExtra = { confirmed: true };
2383
+ return null;
2384
+ });
2385
+ return granted && answer.isError !== true ? appendNote(answer, "Note: a person allowed this.") : answer;
2386
+ }
2387
+ /**
2388
+ * The policy's verdict on a touch about to happen: stop with an answer (blocked, needs a person's OK, or
2389
+ * the check couldn't be made: fail closed), or go on (`granted` when a person's OK is being used up).
2390
+ */
2391
+ async policyGate(session, tool, method, params, lists) {
2392
+ if (lists.confirm.length === 0 && lists.block.length === 0)
2393
+ return { granted: false };
2394
+ const resolved = await this.resolveForPolicy(session, method, params);
2395
+ if (resolved.kind === "error") {
2396
+ // Fail closed: what can't be checked isn't touched. The answer is the one the touch would give.
2397
+ this.showAction("agent", method, undefined, params, resolved.code, resolved.reason);
2398
+ return { stop: fail(resolved.text) };
2399
+ }
2400
+ const verdict = this.policyWords(lists, resolved.elements);
2401
+ if (verdict.kind === "blocked") {
2402
+ this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "blocked");
2403
+ return { stop: fail(this.blockedLine(verdict.element, verdict.word)) };
2404
+ }
2405
+ if (verdict.kind === "none")
2406
+ return { granted: false };
2407
+ const label = verdict.element.label ?? verdict.element.text ?? undefined;
2408
+ const caption = captionFor(tool, label ?? undefined, params) ?? tool;
2409
+ // An OK is for this app, this tool and this element only (Tapwright's review, M1).
2410
+ const check = this.confirmations.check(`${session.app.id}|${tool}|${elementKey(verdict.element)}`, { kind: tool, caption, word: verdict.word });
2411
+ if (check.granted)
2412
+ return { granted: true };
2413
+ const name = this.shortName(verdict.element, false);
2414
+ if ("held" in check) {
2415
+ this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "needs_confirmation");
2416
+ return {
2417
+ stop: fail(`Error not_allowed: needs_confirmation: ${name} needs a person's OK (policy word ${JSON.stringify(verdict.word)}), and they declined it just now. Don't ask again by repeating the call: ask your user what they want.`),
2418
+ };
2419
+ }
2420
+ const where = this.options.canConfirm === false
2421
+ ? "The live view is off, so nobody can allow it here; ask your user."
2422
+ : "The live view asks them now; when they allow it, make the same call again. If nobody is watching, ask your user.";
2423
+ this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "needs_confirmation");
2424
+ return {
2425
+ stop: fail(`Error not_allowed: needs_confirmation: ${this.shortName(verdict.element, false)} needs a person's OK (policy word ${JSON.stringify(verdict.word)}). ${where}${check.suffix}`),
2426
+ };
2427
+ }
2428
+ /** Which policy word, if any, the elements the target could mean carry: a block word first, then a confirm word. */
2429
+ policyWords(lists, elements) {
2430
+ for (const element of elements) {
2431
+ const word = matchWord(lists.block, element);
2432
+ if (word !== null)
2433
+ return { kind: "blocked", element, word };
2434
+ }
2435
+ for (const element of elements) {
2436
+ const word = matchWord(lists.confirm, element);
2437
+ if (word !== null)
2438
+ return { kind: "confirm", element, word };
2439
+ }
2440
+ return { kind: "none" };
2441
+ }
2442
+ blockedLine(element, word) {
2443
+ return `Error not_allowed: blocked: ${this.shortName(element, false)} is blocked by the policy (${JSON.stringify(word)}); don't try another way.`;
2444
+ }
2445
+ /** A dry run the app answers itself (scroll, back, swipe): its checks, nothing touched, one line. */
2446
+ async dryCall(method, params, line) {
2447
+ const session = this.host.current();
2448
+ if (session === undefined)
2449
+ return this.notConnected();
2450
+ if (!session.capabilities.includes("dryRun")) {
2451
+ return fail("Error unsupported: this app's Tapline is too old for dry runs; update react-native-tapline.");
2452
+ }
2453
+ const stale = this.staleRef(params);
2454
+ if (stale !== null)
2455
+ return stale;
2456
+ this.actionStartedAt = Date.now();
2457
+ let result;
2458
+ try {
2459
+ result = await this.host.request(method, { ...params, dryRun: true }, { sessionId: session.id });
2460
+ }
2461
+ catch (error) {
2462
+ this.logExtra = { dry_run: true };
2463
+ this.showAction("agent", method, undefined, params, error instanceof TaplineError ? error.code : "internal", error instanceof TaplineError ? reasonOf(error.code, error.message) : undefined);
2464
+ return fail(describeError(error));
2465
+ }
2466
+ if (typeof result !== "object" || result === null || result.dryRun !== true) {
2467
+ return fail("Error internal: the app didn't answer the dry run.");
2468
+ }
2469
+ this.logExtra = { dry_run: true };
2470
+ this.showAction("agent", method, result, params);
2471
+ return text([line(result), ...this.takePersonNote()].join("\n"));
2472
+ }
2473
+ /** A dry run of a tool the policy doesn't guard (type_text): the app's checks, nothing touched. */
2474
+ async dryRun(tool, method, params) {
2475
+ const session = this.host.current();
2476
+ if (session === undefined)
2477
+ return this.notConnected();
2478
+ const stale = this.staleRef(params);
2479
+ if (stale !== null)
2480
+ return stale;
2481
+ const resolved = await this.resolveForPolicy(session, method, params);
2482
+ this.logExtra = { dry_run: true };
2483
+ if (resolved.kind === "error") {
2484
+ this.showAction("agent", method, undefined, params, resolved.code, resolved.reason);
2485
+ return fail(resolved.text);
2486
+ }
2487
+ this.showAction("agent", method, resolved.elements[0] !== undefined ? { element: resolved.elements[0] } : undefined, params);
2488
+ return text([this.wouldLine(tool, resolved.elements[0], resolved.point, resolved.value), ...this.takePersonNote()].join("\n"));
2489
+ }
2490
+ /**
2491
+ * What a touch would act on. Apps that know dry runs resolve it themselves, with every check; older
2492
+ * apps only give their element list, and every element the target could mean is checked.
2493
+ */
2494
+ async resolveForPolicy(session, method, params) {
2495
+ if (session.capabilities.includes("dryRun")) {
2496
+ let raw;
2497
+ try {
2498
+ raw = await this.host.request(method, { ...params, dryRun: true }, { sessionId: session.id });
2499
+ }
2500
+ catch (error) {
2501
+ const reason = error instanceof TaplineError ? reasonOf(error.code, error.message) : undefined;
2502
+ return {
2503
+ kind: "error",
2504
+ text: describeError(error),
2505
+ code: error instanceof TaplineError ? error.code : "internal",
2506
+ ...(reason !== undefined ? { reason } : {}),
2507
+ };
2508
+ }
2509
+ const parsed = dryRunResultSchema.safeParse(raw);
2510
+ if (!parsed.success)
2511
+ return { kind: "error", text: "Error internal: the app didn't answer the dry run.", code: "internal" };
2512
+ return {
2513
+ kind: "ok",
2514
+ elements: parsed.data.element ? [parsed.data.element] : [],
2515
+ ...(parsed.data.point !== undefined ? { point: parsed.data.point } : {}),
2516
+ ...(parsed.data.value !== undefined ? { value: parsed.data.value } : {}),
2517
+ };
2518
+ }
2519
+ // An app without dry runs: a swipe is checked by what lies under its start.
2520
+ const target = method === "swipe" ? { x: params["x0"], y: params["y0"] } : params["target"];
2521
+ if (method === "type" || target === undefined) {
2522
+ return { kind: "error", text: "Error unsupported: this app's Tapline is too old for dry runs; update react-native-tapline.", code: "unsupported" };
2523
+ }
2524
+ let raw;
2525
+ try {
2526
+ raw = await this.host.request("tree", {}, { sessionId: session.id });
2527
+ }
2528
+ catch (error) {
2529
+ return { kind: "error", text: describeError(error), code: error instanceof TaplineError ? error.code : "internal" };
2530
+ }
2531
+ const tree = treeResultSchema.safeParse(raw);
2532
+ if (!tree.success)
2533
+ return { kind: "error", text: "Error internal: the app returned an element list the host could not read.", code: "internal" };
2534
+ const elements = candidatesFor(target, tree.data.elements);
2535
+ if (elements.length === 0 && method !== "swipe") {
2536
+ return { kind: "error", text: "Error not_found: nothing on screen matches that target.", code: "not_found" };
2537
+ }
2538
+ const first = elements[0];
2539
+ const frame = elements.length === 1 && first !== undefined ? first.frame : null;
2540
+ return {
2541
+ kind: "ok",
2542
+ elements,
2543
+ ...(frame ? { point: { x: Math.round(frame.x + frame.width / 2), y: Math.round(frame.y + frame.height / 2) } } : {}),
2544
+ };
2545
+ }
2546
+ /** `Would tap e12 button "Delete trip" at 215,640.` */
2547
+ wouldLine(tool, element, point, value) {
2548
+ const name = element !== undefined ? this.shortName(element, true) : "the element";
2549
+ const at = point !== undefined ? ` at ${point.x},${point.y}` : "";
2550
+ switch (tool) {
2551
+ case "long_press":
2552
+ return `Would long-press ${name}${at}.`;
2553
+ case "toggle":
2554
+ return `Would toggle ${name}${typeof value === "boolean" ? ` to ${String(value)}` : ""}${at}.`;
2555
+ case "type_text":
2556
+ return `Would type into ${name}.`;
2557
+ default:
2558
+ return `Would tap ${name}${at}.`;
2559
+ }
2560
+ }
2561
+ /** `e12 button "Delete trip"`: ref (when asked), role and name, nothing else. */
2562
+ shortName(element, withRef) {
2563
+ return formatElement({
2564
+ ref: withRef ? element.ref : "",
2565
+ role: element.role ?? null,
2566
+ type: element.type ?? null,
2567
+ label: element.label ?? null,
2568
+ text: element.text ?? null,
2569
+ textMore: element.textMore ?? null,
2570
+ }, { actions: false });
2571
+ }
2572
+ /** A `{ref}` target after a person used the app: refs may point elsewhere. */
2573
+ staleRef(params) {
2574
+ if (!this.personActed)
2575
+ return null;
2576
+ const target = typeof params === "object" && params !== null ? params.target : undefined;
2577
+ if (typeof target === "object" && target !== null && "ref" in target) {
2578
+ // The error says it already, so the note isn't repeated.
2579
+ this.personNotePending = false;
2580
+ return fail("Error stale: a person used the app since your last look, so refs from that list may point elsewhere. Look again, or target by id, label or text.");
2581
+ }
2582
+ return null;
2583
+ }
1341
2584
  tap(args) {
1342
- return this.act("press", { target: toWireTarget(args.target), ...(args.force === true ? { force: true } : {}) }, (r) => `Tapped ${this.elementName(r)}.`, args.look, args.screenshot);
2585
+ const params = { target: toWireTarget(withIndex(args.target, args.index)), ...(args.force === true ? { force: true } : {}) };
2586
+ return this.guarded("tap", "press", params, args.dry_run === true, (preflight) => this.act("press", params, (r) => `Tapped ${this.elementName(r)}.`, args.look, args.screenshot, undefined, undefined, preflight));
1343
2587
  }
1344
2588
  /** `seconds` is accepted for the shared interface; Tapline calls the handler, so it doesn't hold. */
1345
2589
  longPress(args) {
1346
- return this.act("longPress", { target: toWireTarget(args.target), ...(args.force === true ? { force: true } : {}) }, (r) => `Long-pressed ${this.elementName(r)}.`, args.look, args.screenshot);
2590
+ const params = { target: toWireTarget(withIndex(args.target, args.index)), ...(args.force === true ? { force: true } : {}) };
2591
+ return this.guarded("long_press", "longPress", params, args.dry_run === true, (preflight) => this.act("longPress", params, (r) => `Long-pressed ${this.elementName(r)}.`, args.look, args.screenshot, undefined, undefined, preflight));
1347
2592
  }
1348
2593
  typeText(args) {
1349
2594
  const params = {
@@ -1352,13 +2597,18 @@ export class TaplineTools {
1352
2597
  submit: args.submit ?? false,
1353
2598
  };
1354
2599
  if (args.target !== undefined)
1355
- params["target"] = toWireTarget(args.target);
2600
+ params["target"] = toWireTarget(withIndex(args.target, args.index));
1356
2601
  if (args.force === true)
1357
2602
  params["force"] = true;
2603
+ // Typing isn't guarded by the policy (§11.1: the form's button is); a dry run still checks the field.
2604
+ if (args.dry_run === true)
2605
+ return this.dryRun("type_text", "type", params);
1358
2606
  return this.act("type", params, (r) => {
1359
2607
  const value = typeof r === "object" && r !== null && "value" in r ? r.value : undefined;
1360
- const shown = typeof value === "string" ? ` Value is now ${JSON.stringify(value)}.` : "";
1361
- return `Typed into ${this.elementName(r)}${args.submit ? " and submitted" : ""}.${shown}`;
2608
+ // AGENT-TOOLS 1.8 §3.3: a secure field's value is never shown, not even as bullets in this line
2609
+ const secure = typeof r === "object" && r !== null && "secure" in r && r.secure === true;
2610
+ const shown = secure ? " The value is hidden (secure field)." : typeof value === "string" ? ` Value is now ${JSON.stringify(value)}.` : "";
2611
+ return `Typed into ${this.elementName(r, "element", shown !== "")}${args.submit ? " and submitted" : ""}.${shown}`;
1362
2612
  }, args.look, args.screenshot);
1363
2613
  }
1364
2614
  scroll(args) {
@@ -1375,6 +2625,13 @@ export class TaplineTools {
1375
2625
  const flag = (r, key) => typeof r === "object" && r !== null && key in r ? r[key] : undefined;
1376
2626
  // The element was on screen and clear already: nothing was scrolled, and the first line says so.
1377
2627
  const alreadyThere = (r) => toTarget && flag(r, "unchanged") === true;
2628
+ if (args.dry_run === true) {
2629
+ return this.dryCall("scroll", params, (r) => {
2630
+ if (toTarget)
2631
+ return `Would scroll ${this.elementName(r)} to ${this.elementName(r, "target")}.`;
2632
+ return `Would scroll ${this.elementName(r)} ${args.to !== undefined ? `to ${args.to}` : (args.direction ?? "down")}.`;
2633
+ });
2634
+ }
1378
2635
  const atEndAlready = (r) => (args.to === "start" || args.to === "end") && flag(r, "moved") === false;
1379
2636
  return this.act("scroll", params, (r) => {
1380
2637
  if (alreadyThere(r))
@@ -1410,19 +2667,19 @@ export class TaplineTools {
1410
2667
  });
1411
2668
  }
1412
2669
  toggle(args) {
1413
- const params = { target: toWireTarget(args.target) };
2670
+ const params = { target: toWireTarget(withIndex(args.target, args.index)) };
1414
2671
  if (args.value !== undefined)
1415
2672
  params["value"] = args.value;
1416
2673
  if (args.force === true)
1417
2674
  params["force"] = true;
1418
2675
  // The switch already had the asked value: the app touched nothing, and that line is the whole answer.
1419
2676
  const unchanged = (r) => typeof r === "object" && r !== null && "unchanged" in r && r.unchanged === true;
1420
- return this.act("toggle", params, (r) => {
2677
+ return this.guarded("toggle", "toggle", params, args.dry_run === true, (preflight) => this.act("toggle", params, (r) => {
1421
2678
  const value = typeof r === "object" && r !== null && "value" in r ? r.value : undefined;
1422
2679
  if (unchanged(r))
1423
2680
  return `${this.elementName(r)} is already ${JSON.stringify(value)}.`;
1424
2681
  return `Toggled ${this.elementName(r)}${value !== undefined ? ` to ${JSON.stringify(value)}` : ""}.`;
1425
- }, args.look, args.screenshot, undefined, unchanged);
2682
+ }, args.look, args.screenshot, undefined, unchanged, preflight));
1426
2683
  }
1427
2684
  async appCommand(args) {
1428
2685
  if (args.name === undefined || args.name.trim() === "") {
@@ -1474,12 +2731,30 @@ export class TaplineTools {
1474
2731
  const params = { x0: args.x0, y0: args.y0, x1: args.x1, y1: args.y1 };
1475
2732
  if (args.seconds !== undefined)
1476
2733
  params["seconds"] = args.seconds;
1477
- return this.act("swipe", params, (r) => {
2734
+ if (args.dry_run === true) {
2735
+ return this.dryCall("swipe", params, (r) => {
2736
+ const on = this.elementName(r);
2737
+ return `Would swipe from (${args.x0}, ${args.y0}) to (${args.x1}, ${args.y1})${on === "element" ? "" : ` starting on ${on}`}.`;
2738
+ });
2739
+ }
2740
+ // A swipe that moves less than 10 pt is a touch where it starts: the policy checks it (AGENT-TOOLS 1.8 §11.1).
2741
+ const short = Math.hypot(args.x1 - args.x0, args.y1 - args.y0) < 10;
2742
+ const perform = (preflight) => this.act("swipe", params, (r) => {
1478
2743
  const on = this.elementName(r);
1479
2744
  return `Swiped from (${args.x0}, ${args.y0}) to (${args.x1}, ${args.y1})${on === "element" ? "" : ` on ${on}`}.`;
1480
- }, args.look, args.screenshot);
2745
+ }, args.look, args.screenshot, undefined, undefined, preflight);
2746
+ return short ? this.guarded("swipe", "swipe", params, false, perform) : perform();
1481
2747
  }
1482
2748
  back(args) {
2749
+ if (args.dry_run === true) {
2750
+ return this.dryCall("back", {}, (r) => {
2751
+ if (typeof r === "object" && r !== null && "closed" in r && r.closed === "sheet")
2752
+ return "Would close the sheet.";
2753
+ const route = typeof r === "object" && r !== null && "route" in r ? r.route : undefined;
2754
+ const from = Array.isArray(route) && route.every((name) => typeof name === "string") ? ` from ${route.join(" › ")}` : "";
2755
+ return `Would go back${from}.`;
2756
+ });
2757
+ }
1483
2758
  return this.act("back", {}, (r) => {
1484
2759
  if (typeof r === "object" && r !== null && "closed" in r && r.closed === "sheet")
1485
2760
  return "Closed the sheet.";
@@ -1488,8 +2763,16 @@ export class TaplineTools {
1488
2763
  return `Went back${from}.`;
1489
2764
  }, args.look, args.screenshot);
1490
2765
  }
1491
- openUrl(args) {
1492
- return this.act("openURL", { url: args.url }, () => `Opened ${args.url}.`, args.look, args.screenshot);
2766
+ async openUrl(args) {
2767
+ const answer = await this.act("openURL", { url: args.url }, () => `Opened ${args.url}.`, args.look, args.screenshot);
2768
+ // The app's own forget link unpairs it, so it drops the connection before answering: that is the link working.
2769
+ if (answer.isError === true && /(^|\/)tapline\/forget\/?$/.test(args.url.replace(/[?#].*$/, ""))) {
2770
+ const said = answer.content.map((c) => (c.type === "text" ? c.text : "")).join("\n");
2771
+ if (/^Error (reconnecting|not_connected)\b/.test(said)) {
2772
+ return text("Unpaired: the app forgot this computer and disconnected. To connect it again, call `pair`.");
2773
+ }
2774
+ }
2775
+ return answer;
1493
2776
  }
1494
2777
  async logs(args) {
1495
2778
  const session = this.host.current();