tapline-host 1.2.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/tools.js CHANGED
@@ -1,21 +1,24 @@
1
1
  import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
2
+ import { AsyncLocalStorage } from "node:async_hooks";
2
3
  import { basename, dirname, join } from "node:path";
3
4
  import { toBuffer as qrPng } from "qrcode";
4
5
  import { z } from "zod";
5
6
  import { annotateScreenshot } from "./annotate.js";
6
7
  import { agentKind, captionFor, personKind, reasonOf } from "./captions.js";
7
8
  import { composeFilmstrip, diffImages, imageSize } from "./images.js";
8
- import { alertNotes, describeElement, formatChanges, formatElement, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
9
+ import { alertNotes, describeElement, filterLogs, formatChanges, formatElement, formatLogAnswer, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
9
10
  import { formatInspect, inspectResultSchema } from "./inspect.js";
10
11
  import { AUDIT_TYPES, auditResultSchema, auditTitle, formatAudit } from "./audit.js";
12
+ import { NO_CRASH_TEXT, crashNote, disconnectNote, formatCrash } from "./crash.js";
11
13
  import { ActionLog, loggedTarget } from "./actionlog.js";
12
14
  import { Confirmations } from "./confirm.js";
13
15
  import { DEFAULT_POLICY, elementKey, listsFor, matchWord } from "./policy.js";
14
16
  import { readScript, refStep, scriptSteps } from "./script.js";
15
- import { EXPECT_TIMEOUT_S, MAX_STEPS, RUN_TIMEOUT_MS, RUN_TOOLS, describeRunStep, expectFailure, parseError, stepNotes, stepSchema, } from "./run.js";
17
+ import { badUnderKeys, checkUnder, formatPerfLines, perfFirstLine, perfResultSchema } from "./perf.js";
18
+ import { EXPECT_TIMEOUT_S, MAX_STEPS, RUN_TIMEOUT_MS, RUN_TOOLS, UNDER_KEYS, describeRunStep, expectFailure, parseError, stepNotes, stepSchema, } from "./run.js";
16
19
  import { PAIR_CODE_TTL_MS, forgetLink, pairingLink } from "./pairing.js";
17
20
  import { elementSchema, logEntrySchema, screenshotResultSchema, treeResultSchema, } from "./protocol.js";
18
- import { TaplineError } from "./server.js";
21
+ import { TaplineError, identityOf } from "./server.js";
19
22
  // ---- input schemas ----------------------------------------------------------
20
23
  export const targetSchema = z
21
24
  .object({
@@ -230,15 +233,29 @@ export const toolShapes = {
230
233
  },
231
234
  logs: {
232
235
  since: z.number().optional().describe("Only lines after this timestamp (epoch milliseconds)"),
236
+ level: z.enum(["error", "warning", "info"]).optional().describe("That level and above (default info: everything)"),
237
+ grep: z.string().optional().describe("Only lines containing this text (case-insensitive)"),
238
+ limit: z.number().int().min(1).max(500).optional().describe("At most this many lines, the newest (1 to 500, default 100)"),
239
+ },
240
+ crash: {
241
+ all: z.boolean().optional().describe("The last 5 crashes instead of only the newest"),
233
242
  },
234
243
  reload: { look: lookField },
244
+ performance: {
245
+ launch: z
246
+ .boolean()
247
+ .optional()
248
+ .describe("true: reload the app's JavaScript and time it to the first settled screen (development builds only; not a cold start)"),
249
+ seconds: z.number().min(0.5).max(30).optional().describe("Measure for this long, 0.5 to 30 seconds (default 3)"),
250
+ tap: targetField.optional().describe("Tap this first, then measure what it sets off"),
251
+ },
235
252
  wait: { seconds: z.number().min(0.1).max(30).describe("How long to wait, 0.1 to 30 seconds"), screenshot: screenshotField },
236
253
  run: {
237
254
  steps: z
238
255
  .array(stepSchema)
239
256
  .min(1)
240
257
  .max(MAX_STEPS)
241
- .describe('The steps, in order: each is a tool call as an object, {"tool": "tap", "target": {...}, "expect": {...}}. Tools: tap, long_press, type_text, scroll, toggle, back, swipe, open_url, wait, wait_for, app_command, compare_screen (checks, never saves). expect (optional): one of text, id or label, with gone, value and timeout (seconds, default 2); or matches (a reference name, alone or with one of them) with threshold.'),
258
+ .describe('The steps, in order: each is a tool call as an object, {"tool": "tap", "target": {...}, "expect": {...}}. Tools: tap, long_press, type_text, scroll, toggle, back, swipe, open_url, wait, wait_for, app_command, compare_screen (checks, never saves), performance. expect (optional): one of text, id or label, with gone, value and timeout (seconds, default 2); or matches (a reference name, alone or with one of them) with threshold; on a performance step, under: limits such as {"launch_s": 2, "drops": 0, "fps_min": 50, "busy_ms": 200, "js_heap_mb": 80}.'),
242
259
  trace: z.boolean().optional().describe("Also list each step that ran, one line each"),
243
260
  dry_run: z
244
261
  .boolean()
@@ -248,6 +265,20 @@ export const toolShapes = {
248
265
  screenshot: screenshotField,
249
266
  },
250
267
  };
268
+ /**
269
+ * The answer for a call that carries arguments the tool doesn't have (null when all are known). Zod strips unknown
270
+ * keys silently, so `audit {labels: true}` would run and ignore `labels`; this says so instead. Every way in to a
271
+ * tool (the MCP server, and through it the bench) goes through here; script steps are checked by `run` itself.
272
+ */
273
+ export function unknownArgumentError(name, args) {
274
+ const known = Object.keys(toolShapes[name]);
275
+ const unknown = Object.keys(args).filter((key) => !known.includes(key));
276
+ if (unknown.length === 0)
277
+ return null;
278
+ // AGENT-TOOLS 1.11 wording, shared with Tapwright: every unknown name in one sentence.
279
+ const takes = known.length === 0 ? "it takes no arguments" : `it takes ${known.join(", ")}`;
280
+ return fail(`Error bad_params: ${name} doesn't take ${unknown.join(", ")}; ${takes}.`);
281
+ }
251
282
  // ---- compare_screen ---------------------------------------------------------
252
283
  const DEFAULT_MATCH_THRESHOLD = 0.5;
253
284
  const MAX_REGION_LINES = 8;
@@ -500,6 +531,18 @@ function toWireTarget(target) {
500
531
  return withIndex({ text: target.text });
501
532
  return { x: target.x ?? 0, y: target.y ?? 0 };
502
533
  }
534
+ /** A target as the agent wrote it, in words: `id=save`, `"Save"`, `e3`, `120,300`. */
535
+ function targetWords(target) {
536
+ if (target.id !== undefined)
537
+ return `id=${target.id}`;
538
+ if (target.text !== undefined)
539
+ return JSON.stringify(target.text);
540
+ if (target.label !== undefined)
541
+ return JSON.stringify(target.label);
542
+ if (target.ref !== undefined)
543
+ return target.ref;
544
+ return `${target.x ?? 0},${target.y ?? 0}`;
545
+ }
503
546
  function candidateLines(details) {
504
547
  const lines = [];
505
548
  const bareRefs = [];
@@ -622,6 +665,12 @@ function candidatesFor(target, elements) {
622
665
  return picked !== undefined ? [picked] : found;
623
666
  }
624
667
  // ---- tool handlers ----------------------------------------------------------
668
+ /** Every tool whose answer can end with a crash note (all but `crash`, which tells it in full). */
669
+ const CRASH_NOTED_TOOLS = [
670
+ "apps", "openApp", "pair", "look", "run", "inspect", "audit", "checkLayout", "viewer", "compareScreen", "filmstrip",
671
+ "record", "replay", "tap", "longPress", "typeText", "scroll", "toggle", "appCommand", "waitFor", "swipe", "back",
672
+ "openUrl", "logs", "reload", "performance", "wait",
673
+ ];
625
674
  /** The tool handlers behind the MCP server. They only talk to a TaplineHost, so they are easy to test. */
626
675
  export class TaplineTools {
627
676
  host;
@@ -661,12 +710,20 @@ export class TaplineTools {
661
710
  replayContext = null;
662
711
  /** Where the latest failed check of the running batch saved its picture. */
663
712
  diffSaved = null;
713
+ /** What the last `performance` call measured, for the `under` limits of a `run` step. */
714
+ lastPerf;
664
715
  /** The elements the latest comparison painted its ignore list from: a recording saves its mask from the same list. */
665
716
  lastMaskElements;
666
717
  /** The screen size in points from the latest list or overlay, for the live viewer's 0..1 positions. */
667
718
  lastScreen;
668
719
  shotCount = 0;
669
720
  pairWrites = 0;
721
+ /** Per app (by identity): crash events up to this one have been told to the agent. */
722
+ notedCrashSeqs = new Map();
723
+ /** Crash events up to this one happened before this agent's session: an app with no entry above starts here. */
724
+ crashBaseline;
725
+ /** Set inside a tool call (and everything it awaits), so a call within a call (`run`'s steps) adds no notes. */
726
+ noteScope = new AsyncLocalStorage();
670
727
  constructor(host, options) {
671
728
  this.host = host;
672
729
  this.options = options;
@@ -682,6 +739,92 @@ export class TaplineTools {
682
739
  if (this.host.current()?.id === session.id)
683
740
  this.markPersonUsedApp();
684
741
  });
742
+ // Crashes before this agent's session began are not its news.
743
+ this.crashBaseline = host.crashMark();
744
+ for (const name of CRASH_NOTED_TOOLS) {
745
+ const original = Reflect.get(this, name);
746
+ if (typeof original !== "function")
747
+ continue;
748
+ const method = original;
749
+ Reflect.set(this, name, (...args) => this.noting(() => method.apply(this, args)));
750
+ }
751
+ }
752
+ /**
753
+ * Runs a tool call and ends its answer with a note when the app stopped (AGENT-TOOLS 15.2): during this
754
+ * call, or earlier, with its time, once. Calls inside a call (`run`'s steps) add nothing of their own.
755
+ */
756
+ noting(run) {
757
+ // Nesting belongs to the call and what it awaits (not to this object), so calls running side by side each
758
+ // count as outermost, and only a step inside a call is skipped.
759
+ if (this.noteScope.getStore() === true)
760
+ return run();
761
+ const mark = this.host.crashMark();
762
+ const startedWith = this.host.current();
763
+ return this.noteScope.run(true, () => {
764
+ const result = run();
765
+ if (result instanceof Promise)
766
+ return result.then((answer) => this.withPersonNote(this.withCrashNotes(answer, mark, startedWith)));
767
+ return this.withPersonNote(this.withCrashNotes(result, mark, startedWith));
768
+ });
769
+ }
770
+ /** The first call has been answered, so the "needs a person" note has had its turn. */
771
+ personNoteDone = false;
772
+ /**
773
+ * The first answer of the session says what only a person can do, once, when something is pending: the app isn't
774
+ * connected (open it on the phone, same Wi-Fi), and the phone has never paired. Nothing pending: never.
775
+ */
776
+ withPersonNote(answer) {
777
+ if (this.personNoteDone || this.options.pairedBefore === undefined)
778
+ return answer;
779
+ if (this.host.current() !== undefined) {
780
+ this.personNoteDone = true;
781
+ return answer;
782
+ }
783
+ if (answer.content[0]?.type !== "text")
784
+ return answer; // nowhere to put it yet: the next answer says it
785
+ const items = [];
786
+ if (!this.options.pairedBefore())
787
+ items.push("the phone isn't paired yet (call `pair`, then the user scans the QR code or taps Connect)");
788
+ items.push("the app isn't connected (the user opens it on the phone, on the same Wi-Fi as this computer)");
789
+ this.personNoteDone = true;
790
+ return appendNote(answer, `Note: needs a person: ${items.join("; ")}.`);
791
+ }
792
+ /** What was told about this app's crashes so far: the crash events up to this number. */
793
+ notedSeqOf(app) {
794
+ return this.notedCrashSeqs.get(identityOf(app)) ?? this.crashBaseline;
795
+ }
796
+ /**
797
+ * The notes for the app this call used (the one at its start, the one at its end, or, with none connected, the
798
+ * one that dropped last). Other apps' events stay pending until that app is the one being driven.
799
+ */
800
+ withCrashNotes(answer, mark, startedWith) {
801
+ const apps = new Map();
802
+ const current = this.host.current();
803
+ // A call that started on an app answers for that app only: another app that became current meanwhile (the first
804
+ // one dropped) keeps its crashes pending until it is the one being driven (Codex review of 1.3.0).
805
+ const endedWith = startedWith === undefined ? (current ?? this.host.lastDisconnect()?.session) : current !== undefined && identityOf(current) === identityOf(startedWith) ? current : undefined;
806
+ for (const app of [startedWith, endedWith])
807
+ if (app !== undefined)
808
+ apps.set(identityOf(app), app);
809
+ const events = [...apps.values()]
810
+ .flatMap((app) => this.host.crashEventsAfter(this.notedSeqOf(app), app))
811
+ .sort((a, b) => a.seq - b.seq);
812
+ if (events.length === 0)
813
+ return answer;
814
+ const told = this.host.crashMark();
815
+ for (const key of apps.keys())
816
+ this.notedCrashSeqs.set(key, told);
817
+ let noted = answer;
818
+ for (const event of events) {
819
+ // During this call: no time. A crash the phone had kept and sent later says when it happened.
820
+ const during = event.seq > mark;
821
+ // A crash kept by an earlier run of the app (this host had not started yet) says so, with its date.
822
+ const earlier = event.kind === "crash" && event.crash.source === "saved" && event.crash.ts < this.host.startedAt;
823
+ noted = appendNote(noted, event.kind === "crash"
824
+ ? crashNote(event.crash, !(during && event.crash.source === "live"), earlier)
825
+ : disconnectNote(during ? undefined : event.at));
826
+ }
827
+ return noted;
685
828
  }
686
829
  /** A person used the app: the next answer says so once, and `{ref}` targets are stale until the next full list. */
687
830
  markPersonUsedApp() {
@@ -730,6 +873,7 @@ export class TaplineTools {
730
873
  }
731
874
  // -- pair
732
875
  async pair(args) {
876
+ this.personNoteDone = true; // its answer says what the person does; no "call pair" note on it
733
877
  const { url, name } = this.options.connect;
734
878
  // Zero-QR pairing (Tapline 1.1): a development build whose JavaScript comes from this computer finds the
735
879
  // offer itself and asks its person to connect. Open for 10 minutes, one use, local network only.
@@ -897,10 +1041,47 @@ export class TaplineTools {
897
1041
  return fail(`Error bad_params: step ${i + 1}: "${step.tool}" can't be a step (steps act; use it on its own call). Steps can be: ${RUN_TOOLS.join(", ")}.`);
898
1042
  }
899
1043
  const shape = toolShapes[step.tool];
900
- const { tool: _tool, expect: _expect, route: _route, ...params } = step;
1044
+ const { tool: _tool, expect: _expect, route: _route, ...given } = step;
1045
+ // §9.1: a step's own look and screenshot are ignored (the batch answers once), whether its tool has them or not.
1046
+ const params = Object.fromEntries(Object.entries(given).filter(([key]) => !(key === "look" || key === "screenshot") || key in shape));
901
1047
  if (step.tool === "compare_screen" && params.update !== undefined) {
902
1048
  return fail(`Error bad_params: step ${i + 1}: update can't be used in run (a step never saves a reference)`);
903
1049
  }
1050
+ const under = step.expect?.under;
1051
+ if (under !== undefined) {
1052
+ if (step.tool !== "performance") {
1053
+ return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): under goes only on a performance step.`);
1054
+ }
1055
+ const unknown = badUnderKeys(under);
1056
+ if (unknown.length > 0) {
1057
+ return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): under has ${unknown.map((key) => JSON.stringify(key)).join(", ")}; the limits are ${UNDER_KEYS.join(", ")}.`);
1058
+ }
1059
+ if (Object.keys(under).length === 0) {
1060
+ return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): under is empty; give at least one limit (${UNDER_KEYS.join(", ")}).`);
1061
+ }
1062
+ const negative = Object.entries(under).filter(([, limit]) => limit < 0 || !Number.isFinite(limit));
1063
+ if (negative.length > 0) {
1064
+ return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): under limits can't be negative (${negative.map(([key]) => key).join(", ")}).`);
1065
+ }
1066
+ if (under["launch_s"] !== undefined && step.launch !== true) {
1067
+ return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): launch_s goes only on a launch step (performance with launch: true).`);
1068
+ }
1069
+ }
1070
+ // A step's own look and screenshot are ignored (§9.1).
1071
+ if (step.tool === "performance") {
1072
+ delete params["look"];
1073
+ delete params["screenshot"];
1074
+ // Checked before any step runs (and in a dry run), not when this step's turn comes.
1075
+ if (params["launch"] === true && params["tap"] !== undefined) {
1076
+ return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): launch reloads the app and tap taps in it; give one of them.`);
1077
+ }
1078
+ }
1079
+ // AGENT-TOOLS 1.11 §4: the shared wording, with the step first.
1080
+ const known = Object.keys(shape);
1081
+ const stray = Object.keys(params).filter((key) => !known.includes(key));
1082
+ if (stray.length > 0) {
1083
+ return fail(`Error bad_params: step ${i + 1}: ${step.tool} doesn't take ${stray.join(", ")}; ${known.length === 0 ? "it takes no arguments" : `it takes ${known.join(", ")}`}.`);
1084
+ }
904
1085
  const parsed = z.object(shape).strict().safeParse(params);
905
1086
  if (!parsed.success) {
906
1087
  return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): ${parsed.error.issues.map((issue) => `${issue.path.join(".") || "params"}: ${issue.message}`).join("; ")}`);
@@ -927,12 +1108,15 @@ export class TaplineTools {
927
1108
  const trace = [];
928
1109
  let ran = 0;
929
1110
  let stop = null;
1111
+ // The newest log time when the current step began: a failed step shows the app's warnings since then (§15.4).
1112
+ let stepMark;
930
1113
  for (const [i, step] of steps.entries()) {
931
1114
  if (Date.now() - started > RUN_TIMEOUT_MS) {
932
1115
  stop = { index: i, code: "timeout", message: `the batch ran longer than ${RUN_TIMEOUT_MS / 1000} s`, rest: [] };
933
1116
  break;
934
1117
  }
935
1118
  const at = Date.now();
1119
+ stepMark = this.newestLogTs(session.id);
936
1120
  this.batch = { id: batchId, step: i + 1 };
937
1121
  const answer = await this.runStep(step).finally(() => {
938
1122
  this.batch = undefined;
@@ -943,7 +1127,22 @@ export class TaplineTools {
943
1127
  break;
944
1128
  }
945
1129
  notes.push(...stepNotes(said, i + 1));
946
- if (step.expect !== undefined) {
1130
+ if (step.expect?.under !== undefined) {
1131
+ // AGENT-TOOLS §15.3: the limits a performance step must keep; every broken one is listed.
1132
+ const measured = this.lastPerf;
1133
+ if (measured === undefined) {
1134
+ // A limit nobody measured must not pass.
1135
+ stop = { index: i, code: "expect_failed", message: "no measurement to check.", rest: [] };
1136
+ break;
1137
+ }
1138
+ const checked = checkUnder(step.expect.under, measured);
1139
+ notes.push(...checked.notes.map((note) => `Note (step ${i + 1}): ${note}`));
1140
+ if (checked.missed.length > 0) {
1141
+ stop = { index: i, code: "expect_failed", message: `${checked.missed.join("; ")}.`, rest: [] };
1142
+ break;
1143
+ }
1144
+ }
1145
+ else if (step.expect !== undefined) {
947
1146
  const missed = await this.expectMissed(step.expect);
948
1147
  if (missed !== null) {
949
1148
  stop = { index: i, code: "expect_failed", message: `its action ran, but ${missed}`, rest: [] };
@@ -987,28 +1186,59 @@ export class TaplineTools {
987
1186
  : `Error ${stop.code}: Ran ${ran} of ${total} steps; stopped at step ${stop.index + 1} (${describeRunStep(steps[stop.index])}): ${stop.message}`;
988
1187
  const diffNote = stop !== null && this.diffSaved !== null ? [`Note: the difference is saved at ${this.diffSaved}.`] : [];
989
1188
  const first = [head, ...trace, ...(stop?.rest ?? []), ...diffNote].join("\n");
1189
+ const tail = [...(stop !== null ? this.failureLogs(session.id, stop.index + 1, stepMark) : []), ...notes];
990
1190
  let response;
991
1191
  const live = this.host.current();
992
1192
  if (args.look === false || live === undefined) {
993
- response = text([first, ...notes].join("\n"));
1193
+ response = text([first, ...tail].join("\n"));
994
1194
  }
995
1195
  else {
996
1196
  response = await this.withLook(live, first, args.screenshot === true);
997
- if (notes.length > 0) {
1197
+ if (tail.length > 0) {
998
1198
  const [top, ...rest] = response.content;
999
1199
  if (top !== undefined && top.type === "text")
1000
- response = { ...response, content: [{ type: "text", text: [top.text.trimEnd(), ...notes].join("\n") }, ...rest] };
1200
+ response = { ...response, content: [{ type: "text", text: [top.text.trimEnd(), ...tail].join("\n") }, ...rest] };
1001
1201
  }
1002
1202
  }
1003
1203
  return failed ? { ...response, isError: true } : response;
1004
1204
  }
1205
+ /** The newest log time of a session (or of the app's current session), undefined without lines. */
1206
+ newestLogTs(sessionId) {
1207
+ return this.sessionLogs(sessionId).reduce((max, e) => (e.ts !== undefined && (max === undefined || e.ts > max) ? e.ts : max), undefined);
1208
+ }
1209
+ /**
1210
+ * A session's buffered lines, also when it has ended. When the same app on the same phone came back as a new
1211
+ * session, that session's lines follow. Another app's lines never stand in for it (Codex review of 1.3.0).
1212
+ */
1213
+ sessionLogs(sessionId) {
1214
+ const record = this.host.sessionRecord(sessionId);
1215
+ if (record === undefined)
1216
+ return [];
1217
+ const now = this.host.current();
1218
+ if (now === undefined || now.id === sessionId || identityOf(now) !== record.identity)
1219
+ return record.events;
1220
+ try {
1221
+ return [...record.events, ...this.host.logs(now.id)];
1222
+ }
1223
+ catch {
1224
+ return record.events;
1225
+ }
1226
+ }
1227
+ /** AGENT-TOOLS §15.4: `Logs (since step <n>):` and up to 20 lines at warning or above since the step began. */
1228
+ failureLogs(sessionId, step, since) {
1229
+ const lines = filterLogs(this.sessionLogs(sessionId), { since, level: "warning", limit: 20 });
1230
+ return lines.length === 0 ? [] : [`Logs (since step ${step}):`, ...formatLogs(lines).split("\n")];
1231
+ }
1005
1232
  /** `run` with dry_run (§11.4): the steps are checked already; step 1 is resolved, nothing is touched. */
1006
1233
  async dryRunBatch(steps) {
1007
1234
  const total = steps.length;
1008
1235
  const first = steps[0];
1009
1236
  const head = `Would run ${total} step${total === 1 ? "" : "s"}`;
1010
- if (["tap", "long_press", "toggle", "type_text", "scroll", "back", "swipe"].includes(first.tool)) {
1011
- const answer = await this.runStep({ ...first, dry_run: true });
1237
+ // A performance step with a tap: the tap resolves like any touch; nothing is measured (AGENT-TOOLS §15.3).
1238
+ const perfTap = first.tool === "performance" ? first["tap"] : undefined;
1239
+ const probe = perfTap !== undefined ? { tool: "tap", target: perfTap } : first;
1240
+ if (["tap", "long_press", "toggle", "type_text", "scroll", "back", "swipe"].includes(probe.tool)) {
1241
+ const answer = await this.runStep({ ...probe, dry_run: true });
1012
1242
  const said = answer.content.map((block) => (block.type === "text" ? block.text : "")).join("\n").trim();
1013
1243
  const [line, ...rest] = said.split("\n");
1014
1244
  if (answer.isError === true) {
@@ -1018,6 +1248,10 @@ export class TaplineTools {
1018
1248
  const would = (line ?? "").replace(/^Would /, "would ");
1019
1249
  return text([`${head}; step 1 ${would}`, ...rest].join("\n"));
1020
1250
  }
1251
+ if (first.tool === "performance") {
1252
+ const p = first;
1253
+ return text(`${head}; step 1 would ${p["launch"] === true ? "reload the app and time it to the first settled screen" : `measure ${Number(p["seconds"] ?? 3)} s`}.`);
1254
+ }
1021
1255
  return text(`${head}; step 1 would ${describeRunStep(first)}.`);
1022
1256
  }
1023
1257
  /** One step of `run`, as its own tool call without a list. */
@@ -1049,6 +1283,8 @@ export class TaplineTools {
1049
1283
  return this.appCommand(quiet);
1050
1284
  case "compare_screen":
1051
1285
  return this.compareStep(rest);
1286
+ case "performance":
1287
+ return this.performance(rest);
1052
1288
  }
1053
1289
  }
1054
1290
  /** The screen's route names now (one cheap read), or null when the app has no router or didn't answer. */
@@ -2096,11 +2332,19 @@ export class TaplineTools {
2096
2332
  // A screenshot takes a few hundred ms on a phone, so frame 1 is the "before".
2097
2333
  // The tap is a touch like any other: the safety policy decides first.
2098
2334
  const pressParams = { target: toWireTarget(args.tap) };
2099
- const gate = await this.policyGate(session, "tap", "press", pressParams, listsFor(this.policy, session.app));
2100
- if ("stop" in gate)
2101
- return gate.stop;
2102
- frames.push(await this.takeShot(session.id, frameWidth, 0.8));
2103
- await this.host.request("press", pressParams, { sessionId: session.id });
2335
+ // The policy check, the "before" frame and the tap are one turn of the live view's queue: a pause refuses
2336
+ // it, a person's input goes first, and the screen can't change between the check and the tap (PR #4 review).
2337
+ const turn = async () => {
2338
+ const gate = await this.policyGate(session, "tap", "press", pressParams, listsFor(this.policy, session.app));
2339
+ if ("stop" in gate)
2340
+ return gate.stop;
2341
+ frames.push(await this.takeShot(session.id, frameWidth, 0.8));
2342
+ await this.host.request("press", pressParams, { sessionId: session.id });
2343
+ return null;
2344
+ };
2345
+ const stopped = this.options.control ? await this.options.control.runAgent(turn) : await turn();
2346
+ if (stopped !== null)
2347
+ return stopped;
2104
2348
  }
2105
2349
  for (let i = frames.length; i < count; i++) {
2106
2350
  const at = Date.now();
@@ -2790,37 +3034,213 @@ export class TaplineTools {
2790
3034
  // fall through with an empty list
2791
3035
  }
2792
3036
  }
2793
- const last = entries.reduce((max, e) => (e.ts !== undefined && (max === undefined || e.ts > max) ? e.ts : max), undefined);
2794
- const footer = last !== undefined ? `\n(latest ts ${last}; pass since=${last} for newer lines)` : "";
2795
- return text(`${formatLogs(entries)}${footer}`);
3037
+ const newest = entries.reduce((max, e) => (e.ts !== undefined && (max === undefined || e.ts > max) ? e.ts : max), undefined);
3038
+ const query = { since: args.since, level: args.level, grep: args.grep, limit: args.limit };
3039
+ return text(formatLogAnswer(filterLogs(entries, query), query, newest ?? args.since ?? 0));
3040
+ }
3041
+ /** The newest fatal JS error of the app (`all`: the last 5). Reading them tells the agent what the notes did. */
3042
+ crash(args = {}) {
3043
+ // The app being driven, else the one that stopped last. Never every app's crashes at once.
3044
+ const known = this.host.current() ?? this.host.lastCrashEvent()?.of;
3045
+ if (known === undefined)
3046
+ return text(NO_CRASH_TEXT);
3047
+ // Reading them tells the agent what the notes of this app said; other apps' stay pending.
3048
+ this.notedCrashSeqs.set(identityOf(known), this.host.crashMark());
3049
+ const crashes = this.host.crashes(known);
3050
+ if (crashes.length === 0)
3051
+ return text(NO_CRASH_TEXT);
3052
+ return text(crashes.slice(0, args.all === true ? 5 : 1).map(formatCrash).join("\n\n"));
2796
3053
  }
2797
3054
  async reload(args) {
2798
3055
  const session = this.host.current();
2799
3056
  if (session === undefined)
2800
3057
  return this.notConnected();
2801
3058
  try {
2802
- await this.host.request("reload", {}, { sessionId: session.id, timeoutMs: 5000 });
2803
- }
2804
- catch (error) {
2805
- // The app dies mid-reload, so a dropped connection or a missing answer means it worked.
2806
- const worked = error instanceof TaplineError && (error.code === "reconnecting" || error.code === "timeout" || error.code === "not_answering");
2807
- if (!worked)
2808
- return fail(describeError(error));
3059
+ try {
3060
+ // The app goes away on purpose: that is no crash.
3061
+ this.host.expectEnd(session.id);
3062
+ await this.host.request("reload", {}, { sessionId: session.id, timeoutMs: 5000 });
3063
+ }
3064
+ catch (error) {
3065
+ // The app dies mid-reload, so a dropped connection or a missing answer means it worked.
3066
+ const worked = error instanceof TaplineError && (error.code === "reconnecting" || error.code === "timeout" || error.code === "not_answering");
3067
+ if (!worked)
3068
+ return fail(describeError(error));
3069
+ }
3070
+ if (args.look === false)
3071
+ return text("Reload requested.");
3072
+ const deadline = Date.now() + this.reloadWaitMs;
3073
+ for (;;) {
3074
+ const now = this.host.current();
3075
+ if (now !== undefined && now.id !== session.id)
3076
+ break;
3077
+ if (Date.now() > deadline)
3078
+ return text("Reload requested, but the app has not reconnected yet. Try look in a few seconds.");
3079
+ await sleep(100);
3080
+ }
2809
3081
  }
2810
- if (args.look === false)
2811
- return text("Reload requested.");
2812
- const deadline = Date.now() + this.reloadWaitMs;
2813
- for (;;) {
2814
- const now = this.host.current();
2815
- if (now !== undefined && now.id !== session.id)
2816
- break;
2817
- if (Date.now() > deadline)
2818
- return text("Reload requested, but the app has not reconnected yet. Try look in a few seconds.");
2819
- await sleep(100);
3082
+ finally {
3083
+ // Whatever happened, the wait is over: a reload that "worked" without the app going away must not
3084
+ // leave a later, real crash of this session explained.
3085
+ this.host.expectEnd(session.id, false);
2820
3086
  }
2821
3087
  await sleep(this.reloadSettleMs);
2822
3088
  return this.withLook(this.host.current() ?? session, "Reloaded; the app reconnected.");
2823
3089
  }
3090
+ // -- performance (AGENT-TOOLS §15.3)
3091
+ /**
3092
+ * `launch`: a JavaScript reload (development builds), timed from the request to the new session's first
3093
+ * settled screen. Stops with the answer when the reload can't be made or the app doesn't come back.
3094
+ */
3095
+ async timedReload(session) {
3096
+ const started = Date.now();
3097
+ let fresh;
3098
+ try {
3099
+ try {
3100
+ // The app goes away on purpose: that is no crash.
3101
+ this.host.expectEnd(session.id);
3102
+ await this.host.request("reload", {}, { sessionId: session.id, timeoutMs: 5000 });
3103
+ }
3104
+ catch (error) {
3105
+ // The app dies mid-reload, so a dropped connection or a missing answer means it worked.
3106
+ const worked = error instanceof TaplineError && (error.code === "reconnecting" || error.code === "timeout" || error.code === "not_answering");
3107
+ if (!worked) {
3108
+ if (error instanceof TaplineError && error.code === "unsupported") {
3109
+ return { stop: fail(`Error unsupported: launch is a JavaScript reload, and only development builds can do one (${error.message}).`) };
3110
+ }
3111
+ return { stop: fail(describeError(error)) };
3112
+ }
3113
+ }
3114
+ const deadline = Date.now() + this.reloadWaitMs;
3115
+ // Only the same app on the same phone, back as a new session, counts: another connected app that became
3116
+ // current meanwhile is never measured as the reload (Codex review of 1.3.0).
3117
+ const same = identityOf(session);
3118
+ while (fresh === undefined) {
3119
+ const back = this.host.sessions().find((s) => s.id !== session.id && identityOf(s) === same);
3120
+ if (back !== undefined) {
3121
+ fresh = back;
3122
+ // and the agent goes on driving it, not whichever app the host picked when the old session dropped
3123
+ if (this.host.current()?.id !== back.id)
3124
+ this.host.select(back.id);
3125
+ }
3126
+ else if (Date.now() > deadline) {
3127
+ return { stop: fail(`Error timeout: the app hadn't reconnected ${Number((this.reloadWaitMs / 1000).toFixed(1))} s after the reload; try look in a few seconds.`) };
3128
+ }
3129
+ else
3130
+ await sleep(100);
3131
+ }
3132
+ }
3133
+ finally {
3134
+ // The wait is over, whatever happened: a reload that "worked" without the app going away must not leave
3135
+ // a later crash of this session explained away.
3136
+ this.host.expectEnd(session.id, false);
3137
+ }
3138
+ // The first settled screen: the app waits until React hasn't committed for a moment (§6).
3139
+ // The first settled screen: a launch time is only reported when the app said it settled.
3140
+ try {
3141
+ await this.host.request("perfSettle", {}, { sessionId: fresh.id, timeoutMs: 8000 });
3142
+ }
3143
+ catch (error) {
3144
+ return { stop: fail(describeError(error)) };
3145
+ }
3146
+ return { session: fresh, seconds: (Date.now() - started) / 1000 };
3147
+ }
3148
+ /** One measuring window in the app: start, (the tap,) wait out `seconds`, stop. */
3149
+ async measureWindow(session, seconds, press) {
3150
+ const id = { sessionId: session.id };
3151
+ // The window's number: the stop names it, so a call never takes another call's numbers (an older SDK has none).
3152
+ let stopParams = {};
3153
+ try {
3154
+ const started = await this.host.request("perfStart", {}, id);
3155
+ const windowId = typeof started === "object" && started !== null ? started.id : undefined;
3156
+ if (typeof windowId === "number")
3157
+ stopParams = { id: windowId };
3158
+ }
3159
+ catch (error) {
3160
+ if (error instanceof TaplineError && error.code === "unsupported") {
3161
+ throw new TaplineError("unsupported", "this app's Tapline is too old for performance; update react-native-tapline.");
3162
+ }
3163
+ throw error;
3164
+ }
3165
+ const started = Date.now();
3166
+ try {
3167
+ // The caller holds the live view's queue for the whole window when there is a tap (see performance).
3168
+ if (press !== undefined)
3169
+ await this.host.request("press", press, id);
3170
+ const rest = seconds * 1000 - (Date.now() - started);
3171
+ if (rest > 0)
3172
+ await sleep(rest);
3173
+ }
3174
+ catch (error) {
3175
+ await this.host.request("perfStop", stopParams, id).catch(() => undefined);
3176
+ throw error;
3177
+ }
3178
+ const parsed = perfResultSchema.safeParse(await this.host.request("perfStop", stopParams, id));
3179
+ if (!parsed.success)
3180
+ throw new TaplineError("internal", "the app returned performance numbers the host could not read.");
3181
+ return parsed.data;
3182
+ }
3183
+ async performance(args = {}) {
3184
+ this.lastPerf = undefined;
3185
+ const session = this.host.current();
3186
+ if (session === undefined)
3187
+ return this.notConnected();
3188
+ if (args.launch === true && args.tap !== undefined) {
3189
+ return fail("Error bad_params: launch reloads the app and tap taps in it; give one of them.");
3190
+ }
3191
+ const seconds = args.seconds ?? 3;
3192
+ try {
3193
+ let measure;
3194
+ if (args.launch === true) {
3195
+ const launched = await this.timedReload(session);
3196
+ if ("stop" in launched)
3197
+ return launched.stop;
3198
+ measure = { result: await this.measureWindow(launched.session, seconds), launchS: launched.seconds };
3199
+ }
3200
+ else {
3201
+ let press;
3202
+ if (args.tap !== undefined) {
3203
+ // The tap is a touch like any other: the safety policy decides first (as for filmstrip).
3204
+ press = { target: toWireTarget(args.tap) };
3205
+ const stale = this.staleRef(press);
3206
+ if (stale !== null)
3207
+ return stale;
3208
+ }
3209
+ // With a tap, the policy check and the whole window are one turn of the live view's queue: a pause refuses
3210
+ // it, a person's input goes first and waits for the window, and the screen can't change between the check
3211
+ // and the tap (Codex and CodeRabbit reviews of 1.3.0).
3212
+ const tapped = press;
3213
+ const window = async () => {
3214
+ if (tapped !== undefined) {
3215
+ const gate = await this.policyGate(session, "tap", "press", tapped, listsFor(this.policy, session.app));
3216
+ if ("stop" in gate)
3217
+ return { stop: gate.stop };
3218
+ }
3219
+ return { result: await this.measureWindow(session, seconds, tapped) };
3220
+ };
3221
+ const outcome = tapped !== undefined && this.options.control ? await this.options.control.runAgent(window) : await window();
3222
+ if ("stop" in outcome)
3223
+ return outcome.stop;
3224
+ measure = { result: outcome.result };
3225
+ }
3226
+ this.lastPerf = measure;
3227
+ if (this.recording !== null && !this.replaying) {
3228
+ this.recording.steps.push({
3229
+ method: "performance",
3230
+ params: {
3231
+ ...(args.launch === true ? { launch: true } : {}),
3232
+ ...(args.seconds !== undefined ? { seconds: args.seconds } : {}),
3233
+ ...(args.tap !== undefined ? { tap: args.tap } : {}),
3234
+ },
3235
+ });
3236
+ }
3237
+ const tapWords = args.tap === undefined ? undefined : targetWords(args.tap);
3238
+ return text([perfFirstLine(measure, tapWords), ...formatPerfLines(measure.result), ...this.takePersonNote()].join("\n"));
3239
+ }
3240
+ catch (error) {
3241
+ return fail(describeError(error));
3242
+ }
3243
+ }
2824
3244
  async wait(args) {
2825
3245
  const session = this.host.current();
2826
3246
  if (session === undefined)