tapline-host 1.2.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit.d.ts +10 -0
- package/dist/audit.js +19 -1
- package/dist/audit.js.map +1 -1
- package/dist/cli.js +9 -4
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +10 -0
- package/dist/config.js +42 -1
- package/dist/config.js.map +1 -1
- package/dist/crash.d.ts +24 -0
- package/dist/crash.js +91 -0
- package/dist/crash.js.map +1 -0
- package/dist/doctor.d.ts +11 -1
- package/dist/doctor.js +74 -2
- package/dist/doctor.js.map +1 -1
- package/dist/format.d.ts +20 -1
- package/dist/format.js +55 -5
- package/dist/format.js.map +1 -1
- package/dist/mcp.js +101 -55
- package/dist/mcp.js.map +1 -1
- package/dist/perf.d.ts +42 -0
- package/dist/perf.js +95 -0
- package/dist/perf.js.map +1 -0
- package/dist/protocol.d.ts +13 -1
- package/dist/protocol.js +11 -0
- package/dist/protocol.js.map +1 -1
- package/dist/run.d.ts +7 -1
- package/dist/run.js +17 -5
- package/dist/run.js.map +1 -1
- package/dist/script.d.ts +6 -0
- package/dist/script.js +35 -3
- package/dist/script.js.map +1 -1
- package/dist/server.d.ts +64 -0
- package/dist/server.js +145 -5
- package/dist/server.js.map +1 -1
- package/dist/tool-text-data.d.ts +3 -1
- package/dist/tool-text-data.js +3 -1
- package/dist/tool-text-data.js.map +1 -1
- package/dist/tools.d.ts +83 -0
- package/dist/tools.js +456 -36
- package/dist/tools.js.map +1 -1
- package/package.json +1 -1
- package/skills/tapline/SKILL.md +3 -2
package/dist/tools.js
CHANGED
|
@@ -1,21 +1,24 @@
|
|
|
1
1
|
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
3
|
import { basename, dirname, join } from "node:path";
|
|
3
4
|
import { toBuffer as qrPng } from "qrcode";
|
|
4
5
|
import { z } from "zod";
|
|
5
6
|
import { annotateScreenshot } from "./annotate.js";
|
|
6
7
|
import { agentKind, captionFor, personKind, reasonOf } from "./captions.js";
|
|
7
8
|
import { composeFilmstrip, diffImages, imageSize } from "./images.js";
|
|
8
|
-
import { alertNotes, describeElement, formatChanges, formatElement, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
|
|
9
|
+
import { alertNotes, describeElement, filterLogs, formatChanges, formatElement, formatLogAnswer, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
|
|
9
10
|
import { formatInspect, inspectResultSchema } from "./inspect.js";
|
|
10
11
|
import { AUDIT_TYPES, auditResultSchema, auditTitle, formatAudit } from "./audit.js";
|
|
12
|
+
import { NO_CRASH_TEXT, crashNote, disconnectNote, formatCrash } from "./crash.js";
|
|
11
13
|
import { ActionLog, loggedTarget } from "./actionlog.js";
|
|
12
14
|
import { Confirmations } from "./confirm.js";
|
|
13
15
|
import { DEFAULT_POLICY, elementKey, listsFor, matchWord } from "./policy.js";
|
|
14
16
|
import { readScript, refStep, scriptSteps } from "./script.js";
|
|
15
|
-
import {
|
|
17
|
+
import { badUnderKeys, checkUnder, formatPerfLines, perfFirstLine, perfResultSchema } from "./perf.js";
|
|
18
|
+
import { EXPECT_TIMEOUT_S, MAX_STEPS, RUN_TIMEOUT_MS, RUN_TOOLS, UNDER_KEYS, describeRunStep, expectFailure, parseError, stepNotes, stepSchema, } from "./run.js";
|
|
16
19
|
import { PAIR_CODE_TTL_MS, forgetLink, pairingLink } from "./pairing.js";
|
|
17
20
|
import { elementSchema, logEntrySchema, screenshotResultSchema, treeResultSchema, } from "./protocol.js";
|
|
18
|
-
import { TaplineError } from "./server.js";
|
|
21
|
+
import { TaplineError, identityOf } from "./server.js";
|
|
19
22
|
// ---- input schemas ----------------------------------------------------------
|
|
20
23
|
export const targetSchema = z
|
|
21
24
|
.object({
|
|
@@ -230,15 +233,29 @@ export const toolShapes = {
|
|
|
230
233
|
},
|
|
231
234
|
logs: {
|
|
232
235
|
since: z.number().optional().describe("Only lines after this timestamp (epoch milliseconds)"),
|
|
236
|
+
level: z.enum(["error", "warning", "info"]).optional().describe("That level and above (default info: everything)"),
|
|
237
|
+
grep: z.string().optional().describe("Only lines containing this text (case-insensitive)"),
|
|
238
|
+
limit: z.number().int().min(1).max(500).optional().describe("At most this many lines, the newest (1 to 500, default 100)"),
|
|
239
|
+
},
|
|
240
|
+
crash: {
|
|
241
|
+
all: z.boolean().optional().describe("The last 5 crashes instead of only the newest"),
|
|
233
242
|
},
|
|
234
243
|
reload: { look: lookField },
|
|
244
|
+
performance: {
|
|
245
|
+
launch: z
|
|
246
|
+
.boolean()
|
|
247
|
+
.optional()
|
|
248
|
+
.describe("true: reload the app's JavaScript and time it to the first settled screen (development builds only; not a cold start)"),
|
|
249
|
+
seconds: z.number().min(0.5).max(30).optional().describe("Measure for this long, 0.5 to 30 seconds (default 3)"),
|
|
250
|
+
tap: targetField.optional().describe("Tap this first, then measure what it sets off"),
|
|
251
|
+
},
|
|
235
252
|
wait: { seconds: z.number().min(0.1).max(30).describe("How long to wait, 0.1 to 30 seconds"), screenshot: screenshotField },
|
|
236
253
|
run: {
|
|
237
254
|
steps: z
|
|
238
255
|
.array(stepSchema)
|
|
239
256
|
.min(1)
|
|
240
257
|
.max(MAX_STEPS)
|
|
241
|
-
.describe('The steps, in order: each is a tool call as an object, {"tool": "tap", "target": {...}, "expect": {...}}. Tools: tap, long_press, type_text, scroll, toggle, back, swipe, open_url, wait, wait_for, app_command, compare_screen (checks, never saves). expect (optional): one of text, id or label, with gone, value and timeout (seconds, default 2); or matches (a reference name, alone or with one of them) with threshold.'),
|
|
258
|
+
.describe('The steps, in order: each is a tool call as an object, {"tool": "tap", "target": {...}, "expect": {...}}. Tools: tap, long_press, type_text, scroll, toggle, back, swipe, open_url, wait, wait_for, app_command, compare_screen (checks, never saves), performance. expect (optional): one of text, id or label, with gone, value and timeout (seconds, default 2); or matches (a reference name, alone or with one of them) with threshold; on a performance step, under: limits such as {"launch_s": 2, "drops": 0, "fps_min": 50, "busy_ms": 200, "js_heap_mb": 80}.'),
|
|
242
259
|
trace: z.boolean().optional().describe("Also list each step that ran, one line each"),
|
|
243
260
|
dry_run: z
|
|
244
261
|
.boolean()
|
|
@@ -248,6 +265,20 @@ export const toolShapes = {
|
|
|
248
265
|
screenshot: screenshotField,
|
|
249
266
|
},
|
|
250
267
|
};
|
|
268
|
+
/**
|
|
269
|
+
* The answer for a call that carries arguments the tool doesn't have (null when all are known). Zod strips unknown
|
|
270
|
+
* keys silently, so `audit {labels: true}` would run and ignore `labels`; this says so instead. Every way in to a
|
|
271
|
+
* tool (the MCP server, and through it the bench) goes through here; script steps are checked by `run` itself.
|
|
272
|
+
*/
|
|
273
|
+
export function unknownArgumentError(name, args) {
|
|
274
|
+
const known = Object.keys(toolShapes[name]);
|
|
275
|
+
const unknown = Object.keys(args).filter((key) => !known.includes(key));
|
|
276
|
+
if (unknown.length === 0)
|
|
277
|
+
return null;
|
|
278
|
+
// AGENT-TOOLS 1.11 wording, shared with Tapwright: every unknown name in one sentence.
|
|
279
|
+
const takes = known.length === 0 ? "it takes no arguments" : `it takes ${known.join(", ")}`;
|
|
280
|
+
return fail(`Error bad_params: ${name} doesn't take ${unknown.join(", ")}; ${takes}.`);
|
|
281
|
+
}
|
|
251
282
|
// ---- compare_screen ---------------------------------------------------------
|
|
252
283
|
const DEFAULT_MATCH_THRESHOLD = 0.5;
|
|
253
284
|
const MAX_REGION_LINES = 8;
|
|
@@ -500,6 +531,18 @@ function toWireTarget(target) {
|
|
|
500
531
|
return withIndex({ text: target.text });
|
|
501
532
|
return { x: target.x ?? 0, y: target.y ?? 0 };
|
|
502
533
|
}
|
|
534
|
+
/** A target as the agent wrote it, in words: `id=save`, `"Save"`, `e3`, `120,300`. */
|
|
535
|
+
function targetWords(target) {
|
|
536
|
+
if (target.id !== undefined)
|
|
537
|
+
return `id=${target.id}`;
|
|
538
|
+
if (target.text !== undefined)
|
|
539
|
+
return JSON.stringify(target.text);
|
|
540
|
+
if (target.label !== undefined)
|
|
541
|
+
return JSON.stringify(target.label);
|
|
542
|
+
if (target.ref !== undefined)
|
|
543
|
+
return target.ref;
|
|
544
|
+
return `${target.x ?? 0},${target.y ?? 0}`;
|
|
545
|
+
}
|
|
503
546
|
function candidateLines(details) {
|
|
504
547
|
const lines = [];
|
|
505
548
|
const bareRefs = [];
|
|
@@ -622,6 +665,12 @@ function candidatesFor(target, elements) {
|
|
|
622
665
|
return picked !== undefined ? [picked] : found;
|
|
623
666
|
}
|
|
624
667
|
// ---- tool handlers ----------------------------------------------------------
|
|
668
|
+
/** Every tool whose answer can end with a crash note (all but `crash`, which tells it in full). */
|
|
669
|
+
const CRASH_NOTED_TOOLS = [
|
|
670
|
+
"apps", "openApp", "pair", "look", "run", "inspect", "audit", "checkLayout", "viewer", "compareScreen", "filmstrip",
|
|
671
|
+
"record", "replay", "tap", "longPress", "typeText", "scroll", "toggle", "appCommand", "waitFor", "swipe", "back",
|
|
672
|
+
"openUrl", "logs", "reload", "performance", "wait",
|
|
673
|
+
];
|
|
625
674
|
/** The tool handlers behind the MCP server. They only talk to a TaplineHost, so they are easy to test. */
|
|
626
675
|
export class TaplineTools {
|
|
627
676
|
host;
|
|
@@ -661,12 +710,20 @@ export class TaplineTools {
|
|
|
661
710
|
replayContext = null;
|
|
662
711
|
/** Where the latest failed check of the running batch saved its picture. */
|
|
663
712
|
diffSaved = null;
|
|
713
|
+
/** What the last `performance` call measured, for the `under` limits of a `run` step. */
|
|
714
|
+
lastPerf;
|
|
664
715
|
/** The elements the latest comparison painted its ignore list from: a recording saves its mask from the same list. */
|
|
665
716
|
lastMaskElements;
|
|
666
717
|
/** The screen size in points from the latest list or overlay, for the live viewer's 0..1 positions. */
|
|
667
718
|
lastScreen;
|
|
668
719
|
shotCount = 0;
|
|
669
720
|
pairWrites = 0;
|
|
721
|
+
/** Per app (by identity): crash events up to this one have been told to the agent. */
|
|
722
|
+
notedCrashSeqs = new Map();
|
|
723
|
+
/** Crash events up to this one happened before this agent's session: an app with no entry above starts here. */
|
|
724
|
+
crashBaseline;
|
|
725
|
+
/** Set inside a tool call (and everything it awaits), so a call within a call (`run`'s steps) adds no notes. */
|
|
726
|
+
noteScope = new AsyncLocalStorage();
|
|
670
727
|
constructor(host, options) {
|
|
671
728
|
this.host = host;
|
|
672
729
|
this.options = options;
|
|
@@ -682,6 +739,92 @@ export class TaplineTools {
|
|
|
682
739
|
if (this.host.current()?.id === session.id)
|
|
683
740
|
this.markPersonUsedApp();
|
|
684
741
|
});
|
|
742
|
+
// Crashes before this agent's session began are not its news.
|
|
743
|
+
this.crashBaseline = host.crashMark();
|
|
744
|
+
for (const name of CRASH_NOTED_TOOLS) {
|
|
745
|
+
const original = Reflect.get(this, name);
|
|
746
|
+
if (typeof original !== "function")
|
|
747
|
+
continue;
|
|
748
|
+
const method = original;
|
|
749
|
+
Reflect.set(this, name, (...args) => this.noting(() => method.apply(this, args)));
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
/**
|
|
753
|
+
* Runs a tool call and ends its answer with a note when the app stopped (AGENT-TOOLS 15.2): during this
|
|
754
|
+
* call, or earlier, with its time, once. Calls inside a call (`run`'s steps) add nothing of their own.
|
|
755
|
+
*/
|
|
756
|
+
noting(run) {
|
|
757
|
+
// Nesting belongs to the call and what it awaits (not to this object), so calls running side by side each
|
|
758
|
+
// count as outermost, and only a step inside a call is skipped.
|
|
759
|
+
if (this.noteScope.getStore() === true)
|
|
760
|
+
return run();
|
|
761
|
+
const mark = this.host.crashMark();
|
|
762
|
+
const startedWith = this.host.current();
|
|
763
|
+
return this.noteScope.run(true, () => {
|
|
764
|
+
const result = run();
|
|
765
|
+
if (result instanceof Promise)
|
|
766
|
+
return result.then((answer) => this.withPersonNote(this.withCrashNotes(answer, mark, startedWith)));
|
|
767
|
+
return this.withPersonNote(this.withCrashNotes(result, mark, startedWith));
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
/** The first call has been answered, so the "needs a person" note has had its turn. */
|
|
771
|
+
personNoteDone = false;
|
|
772
|
+
/**
|
|
773
|
+
* The first answer of the session says what only a person can do, once, when something is pending: the app isn't
|
|
774
|
+
* connected (open it on the phone, same Wi-Fi), and the phone has never paired. Nothing pending: never.
|
|
775
|
+
*/
|
|
776
|
+
withPersonNote(answer) {
|
|
777
|
+
if (this.personNoteDone || this.options.pairedBefore === undefined)
|
|
778
|
+
return answer;
|
|
779
|
+
if (this.host.current() !== undefined) {
|
|
780
|
+
this.personNoteDone = true;
|
|
781
|
+
return answer;
|
|
782
|
+
}
|
|
783
|
+
if (answer.content[0]?.type !== "text")
|
|
784
|
+
return answer; // nowhere to put it yet: the next answer says it
|
|
785
|
+
const items = [];
|
|
786
|
+
if (!this.options.pairedBefore())
|
|
787
|
+
items.push("the phone isn't paired yet (call `pair`, then the user scans the QR code or taps Connect)");
|
|
788
|
+
items.push("the app isn't connected (the user opens it on the phone, on the same Wi-Fi as this computer)");
|
|
789
|
+
this.personNoteDone = true;
|
|
790
|
+
return appendNote(answer, `Note: needs a person: ${items.join("; ")}.`);
|
|
791
|
+
}
|
|
792
|
+
/** What was told about this app's crashes so far: the crash events up to this number. */
|
|
793
|
+
notedSeqOf(app) {
|
|
794
|
+
return this.notedCrashSeqs.get(identityOf(app)) ?? this.crashBaseline;
|
|
795
|
+
}
|
|
796
|
+
/**
|
|
797
|
+
* The notes for the app this call used (the one at its start, the one at its end, or, with none connected, the
|
|
798
|
+
* one that dropped last). Other apps' events stay pending until that app is the one being driven.
|
|
799
|
+
*/
|
|
800
|
+
withCrashNotes(answer, mark, startedWith) {
|
|
801
|
+
const apps = new Map();
|
|
802
|
+
const current = this.host.current();
|
|
803
|
+
// A call that started on an app answers for that app only: another app that became current meanwhile (the first
|
|
804
|
+
// one dropped) keeps its crashes pending until it is the one being driven (Codex review of 1.3.0).
|
|
805
|
+
const endedWith = startedWith === undefined ? (current ?? this.host.lastDisconnect()?.session) : current !== undefined && identityOf(current) === identityOf(startedWith) ? current : undefined;
|
|
806
|
+
for (const app of [startedWith, endedWith])
|
|
807
|
+
if (app !== undefined)
|
|
808
|
+
apps.set(identityOf(app), app);
|
|
809
|
+
const events = [...apps.values()]
|
|
810
|
+
.flatMap((app) => this.host.crashEventsAfter(this.notedSeqOf(app), app))
|
|
811
|
+
.sort((a, b) => a.seq - b.seq);
|
|
812
|
+
if (events.length === 0)
|
|
813
|
+
return answer;
|
|
814
|
+
const told = this.host.crashMark();
|
|
815
|
+
for (const key of apps.keys())
|
|
816
|
+
this.notedCrashSeqs.set(key, told);
|
|
817
|
+
let noted = answer;
|
|
818
|
+
for (const event of events) {
|
|
819
|
+
// During this call: no time. A crash the phone had kept and sent later says when it happened.
|
|
820
|
+
const during = event.seq > mark;
|
|
821
|
+
// A crash kept by an earlier run of the app (this host had not started yet) says so, with its date.
|
|
822
|
+
const earlier = event.kind === "crash" && event.crash.source === "saved" && event.crash.ts < this.host.startedAt;
|
|
823
|
+
noted = appendNote(noted, event.kind === "crash"
|
|
824
|
+
? crashNote(event.crash, !(during && event.crash.source === "live"), earlier)
|
|
825
|
+
: disconnectNote(during ? undefined : event.at));
|
|
826
|
+
}
|
|
827
|
+
return noted;
|
|
685
828
|
}
|
|
686
829
|
/** A person used the app: the next answer says so once, and `{ref}` targets are stale until the next full list. */
|
|
687
830
|
markPersonUsedApp() {
|
|
@@ -730,6 +873,7 @@ export class TaplineTools {
|
|
|
730
873
|
}
|
|
731
874
|
// -- pair
|
|
732
875
|
async pair(args) {
|
|
876
|
+
this.personNoteDone = true; // its answer says what the person does; no "call pair" note on it
|
|
733
877
|
const { url, name } = this.options.connect;
|
|
734
878
|
// Zero-QR pairing (Tapline 1.1): a development build whose JavaScript comes from this computer finds the
|
|
735
879
|
// offer itself and asks its person to connect. Open for 10 minutes, one use, local network only.
|
|
@@ -897,10 +1041,47 @@ export class TaplineTools {
|
|
|
897
1041
|
return fail(`Error bad_params: step ${i + 1}: "${step.tool}" can't be a step (steps act; use it on its own call). Steps can be: ${RUN_TOOLS.join(", ")}.`);
|
|
898
1042
|
}
|
|
899
1043
|
const shape = toolShapes[step.tool];
|
|
900
|
-
const { tool: _tool, expect: _expect, route: _route, ...
|
|
1044
|
+
const { tool: _tool, expect: _expect, route: _route, ...given } = step;
|
|
1045
|
+
// §9.1: a step's own look and screenshot are ignored (the batch answers once), whether its tool has them or not.
|
|
1046
|
+
const params = Object.fromEntries(Object.entries(given).filter(([key]) => !(key === "look" || key === "screenshot") || key in shape));
|
|
901
1047
|
if (step.tool === "compare_screen" && params.update !== undefined) {
|
|
902
1048
|
return fail(`Error bad_params: step ${i + 1}: update can't be used in run (a step never saves a reference)`);
|
|
903
1049
|
}
|
|
1050
|
+
const under = step.expect?.under;
|
|
1051
|
+
if (under !== undefined) {
|
|
1052
|
+
if (step.tool !== "performance") {
|
|
1053
|
+
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): under goes only on a performance step.`);
|
|
1054
|
+
}
|
|
1055
|
+
const unknown = badUnderKeys(under);
|
|
1056
|
+
if (unknown.length > 0) {
|
|
1057
|
+
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): under has ${unknown.map((key) => JSON.stringify(key)).join(", ")}; the limits are ${UNDER_KEYS.join(", ")}.`);
|
|
1058
|
+
}
|
|
1059
|
+
if (Object.keys(under).length === 0) {
|
|
1060
|
+
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): under is empty; give at least one limit (${UNDER_KEYS.join(", ")}).`);
|
|
1061
|
+
}
|
|
1062
|
+
const negative = Object.entries(under).filter(([, limit]) => limit < 0 || !Number.isFinite(limit));
|
|
1063
|
+
if (negative.length > 0) {
|
|
1064
|
+
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): under limits can't be negative (${negative.map(([key]) => key).join(", ")}).`);
|
|
1065
|
+
}
|
|
1066
|
+
if (under["launch_s"] !== undefined && step.launch !== true) {
|
|
1067
|
+
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): launch_s goes only on a launch step (performance with launch: true).`);
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
// A step's own look and screenshot are ignored (§9.1).
|
|
1071
|
+
if (step.tool === "performance") {
|
|
1072
|
+
delete params["look"];
|
|
1073
|
+
delete params["screenshot"];
|
|
1074
|
+
// Checked before any step runs (and in a dry run), not when this step's turn comes.
|
|
1075
|
+
if (params["launch"] === true && params["tap"] !== undefined) {
|
|
1076
|
+
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): launch reloads the app and tap taps in it; give one of them.`);
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
// AGENT-TOOLS 1.11 §4: the shared wording, with the step first.
|
|
1080
|
+
const known = Object.keys(shape);
|
|
1081
|
+
const stray = Object.keys(params).filter((key) => !known.includes(key));
|
|
1082
|
+
if (stray.length > 0) {
|
|
1083
|
+
return fail(`Error bad_params: step ${i + 1}: ${step.tool} doesn't take ${stray.join(", ")}; ${known.length === 0 ? "it takes no arguments" : `it takes ${known.join(", ")}`}.`);
|
|
1084
|
+
}
|
|
904
1085
|
const parsed = z.object(shape).strict().safeParse(params);
|
|
905
1086
|
if (!parsed.success) {
|
|
906
1087
|
return fail(`Error bad_params: step ${i + 1} (${describeRunStep(step)}): ${parsed.error.issues.map((issue) => `${issue.path.join(".") || "params"}: ${issue.message}`).join("; ")}`);
|
|
@@ -927,12 +1108,15 @@ export class TaplineTools {
|
|
|
927
1108
|
const trace = [];
|
|
928
1109
|
let ran = 0;
|
|
929
1110
|
let stop = null;
|
|
1111
|
+
// The newest log time when the current step began: a failed step shows the app's warnings since then (§15.4).
|
|
1112
|
+
let stepMark;
|
|
930
1113
|
for (const [i, step] of steps.entries()) {
|
|
931
1114
|
if (Date.now() - started > RUN_TIMEOUT_MS) {
|
|
932
1115
|
stop = { index: i, code: "timeout", message: `the batch ran longer than ${RUN_TIMEOUT_MS / 1000} s`, rest: [] };
|
|
933
1116
|
break;
|
|
934
1117
|
}
|
|
935
1118
|
const at = Date.now();
|
|
1119
|
+
stepMark = this.newestLogTs(session.id);
|
|
936
1120
|
this.batch = { id: batchId, step: i + 1 };
|
|
937
1121
|
const answer = await this.runStep(step).finally(() => {
|
|
938
1122
|
this.batch = undefined;
|
|
@@ -943,7 +1127,22 @@ export class TaplineTools {
|
|
|
943
1127
|
break;
|
|
944
1128
|
}
|
|
945
1129
|
notes.push(...stepNotes(said, i + 1));
|
|
946
|
-
if (step.expect !== undefined) {
|
|
1130
|
+
if (step.expect?.under !== undefined) {
|
|
1131
|
+
// AGENT-TOOLS §15.3: the limits a performance step must keep; every broken one is listed.
|
|
1132
|
+
const measured = this.lastPerf;
|
|
1133
|
+
if (measured === undefined) {
|
|
1134
|
+
// A limit nobody measured must not pass.
|
|
1135
|
+
stop = { index: i, code: "expect_failed", message: "no measurement to check.", rest: [] };
|
|
1136
|
+
break;
|
|
1137
|
+
}
|
|
1138
|
+
const checked = checkUnder(step.expect.under, measured);
|
|
1139
|
+
notes.push(...checked.notes.map((note) => `Note (step ${i + 1}): ${note}`));
|
|
1140
|
+
if (checked.missed.length > 0) {
|
|
1141
|
+
stop = { index: i, code: "expect_failed", message: `${checked.missed.join("; ")}.`, rest: [] };
|
|
1142
|
+
break;
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
else if (step.expect !== undefined) {
|
|
947
1146
|
const missed = await this.expectMissed(step.expect);
|
|
948
1147
|
if (missed !== null) {
|
|
949
1148
|
stop = { index: i, code: "expect_failed", message: `its action ran, but ${missed}`, rest: [] };
|
|
@@ -987,28 +1186,59 @@ export class TaplineTools {
|
|
|
987
1186
|
: `Error ${stop.code}: Ran ${ran} of ${total} steps; stopped at step ${stop.index + 1} (${describeRunStep(steps[stop.index])}): ${stop.message}`;
|
|
988
1187
|
const diffNote = stop !== null && this.diffSaved !== null ? [`Note: the difference is saved at ${this.diffSaved}.`] : [];
|
|
989
1188
|
const first = [head, ...trace, ...(stop?.rest ?? []), ...diffNote].join("\n");
|
|
1189
|
+
const tail = [...(stop !== null ? this.failureLogs(session.id, stop.index + 1, stepMark) : []), ...notes];
|
|
990
1190
|
let response;
|
|
991
1191
|
const live = this.host.current();
|
|
992
1192
|
if (args.look === false || live === undefined) {
|
|
993
|
-
response = text([first, ...
|
|
1193
|
+
response = text([first, ...tail].join("\n"));
|
|
994
1194
|
}
|
|
995
1195
|
else {
|
|
996
1196
|
response = await this.withLook(live, first, args.screenshot === true);
|
|
997
|
-
if (
|
|
1197
|
+
if (tail.length > 0) {
|
|
998
1198
|
const [top, ...rest] = response.content;
|
|
999
1199
|
if (top !== undefined && top.type === "text")
|
|
1000
|
-
response = { ...response, content: [{ type: "text", text: [top.text.trimEnd(), ...
|
|
1200
|
+
response = { ...response, content: [{ type: "text", text: [top.text.trimEnd(), ...tail].join("\n") }, ...rest] };
|
|
1001
1201
|
}
|
|
1002
1202
|
}
|
|
1003
1203
|
return failed ? { ...response, isError: true } : response;
|
|
1004
1204
|
}
|
|
1205
|
+
/** The newest log time of a session (or of the app's current session), undefined without lines. */
|
|
1206
|
+
newestLogTs(sessionId) {
|
|
1207
|
+
return this.sessionLogs(sessionId).reduce((max, e) => (e.ts !== undefined && (max === undefined || e.ts > max) ? e.ts : max), undefined);
|
|
1208
|
+
}
|
|
1209
|
+
/**
|
|
1210
|
+
* A session's buffered lines, also when it has ended. When the same app on the same phone came back as a new
|
|
1211
|
+
* session, that session's lines follow. Another app's lines never stand in for it (Codex review of 1.3.0).
|
|
1212
|
+
*/
|
|
1213
|
+
sessionLogs(sessionId) {
|
|
1214
|
+
const record = this.host.sessionRecord(sessionId);
|
|
1215
|
+
if (record === undefined)
|
|
1216
|
+
return [];
|
|
1217
|
+
const now = this.host.current();
|
|
1218
|
+
if (now === undefined || now.id === sessionId || identityOf(now) !== record.identity)
|
|
1219
|
+
return record.events;
|
|
1220
|
+
try {
|
|
1221
|
+
return [...record.events, ...this.host.logs(now.id)];
|
|
1222
|
+
}
|
|
1223
|
+
catch {
|
|
1224
|
+
return record.events;
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
/** AGENT-TOOLS §15.4: `Logs (since step <n>):` and up to 20 lines at warning or above since the step began. */
|
|
1228
|
+
failureLogs(sessionId, step, since) {
|
|
1229
|
+
const lines = filterLogs(this.sessionLogs(sessionId), { since, level: "warning", limit: 20 });
|
|
1230
|
+
return lines.length === 0 ? [] : [`Logs (since step ${step}):`, ...formatLogs(lines).split("\n")];
|
|
1231
|
+
}
|
|
1005
1232
|
/** `run` with dry_run (§11.4): the steps are checked already; step 1 is resolved, nothing is touched. */
|
|
1006
1233
|
async dryRunBatch(steps) {
|
|
1007
1234
|
const total = steps.length;
|
|
1008
1235
|
const first = steps[0];
|
|
1009
1236
|
const head = `Would run ${total} step${total === 1 ? "" : "s"}`;
|
|
1010
|
-
|
|
1011
|
-
|
|
1237
|
+
// A performance step with a tap: the tap resolves like any touch; nothing is measured (AGENT-TOOLS §15.3).
|
|
1238
|
+
const perfTap = first.tool === "performance" ? first["tap"] : undefined;
|
|
1239
|
+
const probe = perfTap !== undefined ? { tool: "tap", target: perfTap } : first;
|
|
1240
|
+
if (["tap", "long_press", "toggle", "type_text", "scroll", "back", "swipe"].includes(probe.tool)) {
|
|
1241
|
+
const answer = await this.runStep({ ...probe, dry_run: true });
|
|
1012
1242
|
const said = answer.content.map((block) => (block.type === "text" ? block.text : "")).join("\n").trim();
|
|
1013
1243
|
const [line, ...rest] = said.split("\n");
|
|
1014
1244
|
if (answer.isError === true) {
|
|
@@ -1018,6 +1248,10 @@ export class TaplineTools {
|
|
|
1018
1248
|
const would = (line ?? "").replace(/^Would /, "would ");
|
|
1019
1249
|
return text([`${head}; step 1 ${would}`, ...rest].join("\n"));
|
|
1020
1250
|
}
|
|
1251
|
+
if (first.tool === "performance") {
|
|
1252
|
+
const p = first;
|
|
1253
|
+
return text(`${head}; step 1 would ${p["launch"] === true ? "reload the app and time it to the first settled screen" : `measure ${Number(p["seconds"] ?? 3)} s`}.`);
|
|
1254
|
+
}
|
|
1021
1255
|
return text(`${head}; step 1 would ${describeRunStep(first)}.`);
|
|
1022
1256
|
}
|
|
1023
1257
|
/** One step of `run`, as its own tool call without a list. */
|
|
@@ -1049,6 +1283,8 @@ export class TaplineTools {
|
|
|
1049
1283
|
return this.appCommand(quiet);
|
|
1050
1284
|
case "compare_screen":
|
|
1051
1285
|
return this.compareStep(rest);
|
|
1286
|
+
case "performance":
|
|
1287
|
+
return this.performance(rest);
|
|
1052
1288
|
}
|
|
1053
1289
|
}
|
|
1054
1290
|
/** The screen's route names now (one cheap read), or null when the app has no router or didn't answer. */
|
|
@@ -2096,11 +2332,19 @@ export class TaplineTools {
|
|
|
2096
2332
|
// A screenshot takes a few hundred ms on a phone, so frame 1 is the "before".
|
|
2097
2333
|
// The tap is a touch like any other: the safety policy decides first.
|
|
2098
2334
|
const pressParams = { target: toWireTarget(args.tap) };
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2335
|
+
// The policy check, the "before" frame and the tap are one turn of the live view's queue: a pause refuses
|
|
2336
|
+
// it, a person's input goes first, and the screen can't change between the check and the tap (PR #4 review).
|
|
2337
|
+
const turn = async () => {
|
|
2338
|
+
const gate = await this.policyGate(session, "tap", "press", pressParams, listsFor(this.policy, session.app));
|
|
2339
|
+
if ("stop" in gate)
|
|
2340
|
+
return gate.stop;
|
|
2341
|
+
frames.push(await this.takeShot(session.id, frameWidth, 0.8));
|
|
2342
|
+
await this.host.request("press", pressParams, { sessionId: session.id });
|
|
2343
|
+
return null;
|
|
2344
|
+
};
|
|
2345
|
+
const stopped = this.options.control ? await this.options.control.runAgent(turn) : await turn();
|
|
2346
|
+
if (stopped !== null)
|
|
2347
|
+
return stopped;
|
|
2104
2348
|
}
|
|
2105
2349
|
for (let i = frames.length; i < count; i++) {
|
|
2106
2350
|
const at = Date.now();
|
|
@@ -2790,37 +3034,213 @@ export class TaplineTools {
|
|
|
2790
3034
|
// fall through with an empty list
|
|
2791
3035
|
}
|
|
2792
3036
|
}
|
|
2793
|
-
const
|
|
2794
|
-
const
|
|
2795
|
-
return text(
|
|
3037
|
+
const newest = entries.reduce((max, e) => (e.ts !== undefined && (max === undefined || e.ts > max) ? e.ts : max), undefined);
|
|
3038
|
+
const query = { since: args.since, level: args.level, grep: args.grep, limit: args.limit };
|
|
3039
|
+
return text(formatLogAnswer(filterLogs(entries, query), query, newest ?? args.since ?? 0));
|
|
3040
|
+
}
|
|
3041
|
+
/** The newest fatal JS error of the app (`all`: the last 5). Reading them tells the agent what the notes did. */
|
|
3042
|
+
crash(args = {}) {
|
|
3043
|
+
// The app being driven, else the one that stopped last. Never every app's crashes at once.
|
|
3044
|
+
const known = this.host.current() ?? this.host.lastCrashEvent()?.of;
|
|
3045
|
+
if (known === undefined)
|
|
3046
|
+
return text(NO_CRASH_TEXT);
|
|
3047
|
+
// Reading them tells the agent what the notes of this app said; other apps' stay pending.
|
|
3048
|
+
this.notedCrashSeqs.set(identityOf(known), this.host.crashMark());
|
|
3049
|
+
const crashes = this.host.crashes(known);
|
|
3050
|
+
if (crashes.length === 0)
|
|
3051
|
+
return text(NO_CRASH_TEXT);
|
|
3052
|
+
return text(crashes.slice(0, args.all === true ? 5 : 1).map(formatCrash).join("\n\n"));
|
|
2796
3053
|
}
|
|
2797
3054
|
async reload(args) {
|
|
2798
3055
|
const session = this.host.current();
|
|
2799
3056
|
if (session === undefined)
|
|
2800
3057
|
return this.notConnected();
|
|
2801
3058
|
try {
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
3059
|
+
try {
|
|
3060
|
+
// The app goes away on purpose: that is no crash.
|
|
3061
|
+
this.host.expectEnd(session.id);
|
|
3062
|
+
await this.host.request("reload", {}, { sessionId: session.id, timeoutMs: 5000 });
|
|
3063
|
+
}
|
|
3064
|
+
catch (error) {
|
|
3065
|
+
// The app dies mid-reload, so a dropped connection or a missing answer means it worked.
|
|
3066
|
+
const worked = error instanceof TaplineError && (error.code === "reconnecting" || error.code === "timeout" || error.code === "not_answering");
|
|
3067
|
+
if (!worked)
|
|
3068
|
+
return fail(describeError(error));
|
|
3069
|
+
}
|
|
3070
|
+
if (args.look === false)
|
|
3071
|
+
return text("Reload requested.");
|
|
3072
|
+
const deadline = Date.now() + this.reloadWaitMs;
|
|
3073
|
+
for (;;) {
|
|
3074
|
+
const now = this.host.current();
|
|
3075
|
+
if (now !== undefined && now.id !== session.id)
|
|
3076
|
+
break;
|
|
3077
|
+
if (Date.now() > deadline)
|
|
3078
|
+
return text("Reload requested, but the app has not reconnected yet. Try look in a few seconds.");
|
|
3079
|
+
await sleep(100);
|
|
3080
|
+
}
|
|
2809
3081
|
}
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
const now = this.host.current();
|
|
2815
|
-
if (now !== undefined && now.id !== session.id)
|
|
2816
|
-
break;
|
|
2817
|
-
if (Date.now() > deadline)
|
|
2818
|
-
return text("Reload requested, but the app has not reconnected yet. Try look in a few seconds.");
|
|
2819
|
-
await sleep(100);
|
|
3082
|
+
finally {
|
|
3083
|
+
// Whatever happened, the wait is over: a reload that "worked" without the app going away must not
|
|
3084
|
+
// leave a later, real crash of this session explained.
|
|
3085
|
+
this.host.expectEnd(session.id, false);
|
|
2820
3086
|
}
|
|
2821
3087
|
await sleep(this.reloadSettleMs);
|
|
2822
3088
|
return this.withLook(this.host.current() ?? session, "Reloaded; the app reconnected.");
|
|
2823
3089
|
}
|
|
3090
|
+
// -- performance (AGENT-TOOLS §15.3)
|
|
3091
|
+
/**
|
|
3092
|
+
* `launch`: a JavaScript reload (development builds), timed from the request to the new session's first
|
|
3093
|
+
* settled screen. Stops with the answer when the reload can't be made or the app doesn't come back.
|
|
3094
|
+
*/
|
|
3095
|
+
async timedReload(session) {
|
|
3096
|
+
const started = Date.now();
|
|
3097
|
+
let fresh;
|
|
3098
|
+
try {
|
|
3099
|
+
try {
|
|
3100
|
+
// The app goes away on purpose: that is no crash.
|
|
3101
|
+
this.host.expectEnd(session.id);
|
|
3102
|
+
await this.host.request("reload", {}, { sessionId: session.id, timeoutMs: 5000 });
|
|
3103
|
+
}
|
|
3104
|
+
catch (error) {
|
|
3105
|
+
// The app dies mid-reload, so a dropped connection or a missing answer means it worked.
|
|
3106
|
+
const worked = error instanceof TaplineError && (error.code === "reconnecting" || error.code === "timeout" || error.code === "not_answering");
|
|
3107
|
+
if (!worked) {
|
|
3108
|
+
if (error instanceof TaplineError && error.code === "unsupported") {
|
|
3109
|
+
return { stop: fail(`Error unsupported: launch is a JavaScript reload, and only development builds can do one (${error.message}).`) };
|
|
3110
|
+
}
|
|
3111
|
+
return { stop: fail(describeError(error)) };
|
|
3112
|
+
}
|
|
3113
|
+
}
|
|
3114
|
+
const deadline = Date.now() + this.reloadWaitMs;
|
|
3115
|
+
// Only the same app on the same phone, back as a new session, counts: another connected app that became
|
|
3116
|
+
// current meanwhile is never measured as the reload (Codex review of 1.3.0).
|
|
3117
|
+
const same = identityOf(session);
|
|
3118
|
+
while (fresh === undefined) {
|
|
3119
|
+
const back = this.host.sessions().find((s) => s.id !== session.id && identityOf(s) === same);
|
|
3120
|
+
if (back !== undefined) {
|
|
3121
|
+
fresh = back;
|
|
3122
|
+
// and the agent goes on driving it, not whichever app the host picked when the old session dropped
|
|
3123
|
+
if (this.host.current()?.id !== back.id)
|
|
3124
|
+
this.host.select(back.id);
|
|
3125
|
+
}
|
|
3126
|
+
else if (Date.now() > deadline) {
|
|
3127
|
+
return { stop: fail(`Error timeout: the app hadn't reconnected ${Number((this.reloadWaitMs / 1000).toFixed(1))} s after the reload; try look in a few seconds.`) };
|
|
3128
|
+
}
|
|
3129
|
+
else
|
|
3130
|
+
await sleep(100);
|
|
3131
|
+
}
|
|
3132
|
+
}
|
|
3133
|
+
finally {
|
|
3134
|
+
// The wait is over, whatever happened: a reload that "worked" without the app going away must not leave
|
|
3135
|
+
// a later crash of this session explained away.
|
|
3136
|
+
this.host.expectEnd(session.id, false);
|
|
3137
|
+
}
|
|
3138
|
+
// The first settled screen: the app waits until React hasn't committed for a moment (§6).
|
|
3139
|
+
// The first settled screen: a launch time is only reported when the app said it settled.
|
|
3140
|
+
try {
|
|
3141
|
+
await this.host.request("perfSettle", {}, { sessionId: fresh.id, timeoutMs: 8000 });
|
|
3142
|
+
}
|
|
3143
|
+
catch (error) {
|
|
3144
|
+
return { stop: fail(describeError(error)) };
|
|
3145
|
+
}
|
|
3146
|
+
return { session: fresh, seconds: (Date.now() - started) / 1000 };
|
|
3147
|
+
}
|
|
3148
|
+
/** One measuring window in the app: start, (the tap,) wait out `seconds`, stop. */
|
|
3149
|
+
async measureWindow(session, seconds, press) {
|
|
3150
|
+
const id = { sessionId: session.id };
|
|
3151
|
+
// The window's number: the stop names it, so a call never takes another call's numbers (an older SDK has none).
|
|
3152
|
+
let stopParams = {};
|
|
3153
|
+
try {
|
|
3154
|
+
const started = await this.host.request("perfStart", {}, id);
|
|
3155
|
+
const windowId = typeof started === "object" && started !== null ? started.id : undefined;
|
|
3156
|
+
if (typeof windowId === "number")
|
|
3157
|
+
stopParams = { id: windowId };
|
|
3158
|
+
}
|
|
3159
|
+
catch (error) {
|
|
3160
|
+
if (error instanceof TaplineError && error.code === "unsupported") {
|
|
3161
|
+
throw new TaplineError("unsupported", "this app's Tapline is too old for performance; update react-native-tapline.");
|
|
3162
|
+
}
|
|
3163
|
+
throw error;
|
|
3164
|
+
}
|
|
3165
|
+
const started = Date.now();
|
|
3166
|
+
try {
|
|
3167
|
+
// The caller holds the live view's queue for the whole window when there is a tap (see performance).
|
|
3168
|
+
if (press !== undefined)
|
|
3169
|
+
await this.host.request("press", press, id);
|
|
3170
|
+
const rest = seconds * 1000 - (Date.now() - started);
|
|
3171
|
+
if (rest > 0)
|
|
3172
|
+
await sleep(rest);
|
|
3173
|
+
}
|
|
3174
|
+
catch (error) {
|
|
3175
|
+
await this.host.request("perfStop", stopParams, id).catch(() => undefined);
|
|
3176
|
+
throw error;
|
|
3177
|
+
}
|
|
3178
|
+
const parsed = perfResultSchema.safeParse(await this.host.request("perfStop", stopParams, id));
|
|
3179
|
+
if (!parsed.success)
|
|
3180
|
+
throw new TaplineError("internal", "the app returned performance numbers the host could not read.");
|
|
3181
|
+
return parsed.data;
|
|
3182
|
+
}
|
|
3183
|
+
async performance(args = {}) {
|
|
3184
|
+
this.lastPerf = undefined;
|
|
3185
|
+
const session = this.host.current();
|
|
3186
|
+
if (session === undefined)
|
|
3187
|
+
return this.notConnected();
|
|
3188
|
+
if (args.launch === true && args.tap !== undefined) {
|
|
3189
|
+
return fail("Error bad_params: launch reloads the app and tap taps in it; give one of them.");
|
|
3190
|
+
}
|
|
3191
|
+
const seconds = args.seconds ?? 3;
|
|
3192
|
+
try {
|
|
3193
|
+
let measure;
|
|
3194
|
+
if (args.launch === true) {
|
|
3195
|
+
const launched = await this.timedReload(session);
|
|
3196
|
+
if ("stop" in launched)
|
|
3197
|
+
return launched.stop;
|
|
3198
|
+
measure = { result: await this.measureWindow(launched.session, seconds), launchS: launched.seconds };
|
|
3199
|
+
}
|
|
3200
|
+
else {
|
|
3201
|
+
let press;
|
|
3202
|
+
if (args.tap !== undefined) {
|
|
3203
|
+
// The tap is a touch like any other: the safety policy decides first (as for filmstrip).
|
|
3204
|
+
press = { target: toWireTarget(args.tap) };
|
|
3205
|
+
const stale = this.staleRef(press);
|
|
3206
|
+
if (stale !== null)
|
|
3207
|
+
return stale;
|
|
3208
|
+
}
|
|
3209
|
+
// With a tap, the policy check and the whole window are one turn of the live view's queue: a pause refuses
|
|
3210
|
+
// it, a person's input goes first and waits for the window, and the screen can't change between the check
|
|
3211
|
+
// and the tap (Codex and CodeRabbit reviews of 1.3.0).
|
|
3212
|
+
const tapped = press;
|
|
3213
|
+
const window = async () => {
|
|
3214
|
+
if (tapped !== undefined) {
|
|
3215
|
+
const gate = await this.policyGate(session, "tap", "press", tapped, listsFor(this.policy, session.app));
|
|
3216
|
+
if ("stop" in gate)
|
|
3217
|
+
return { stop: gate.stop };
|
|
3218
|
+
}
|
|
3219
|
+
return { result: await this.measureWindow(session, seconds, tapped) };
|
|
3220
|
+
};
|
|
3221
|
+
const outcome = tapped !== undefined && this.options.control ? await this.options.control.runAgent(window) : await window();
|
|
3222
|
+
if ("stop" in outcome)
|
|
3223
|
+
return outcome.stop;
|
|
3224
|
+
measure = { result: outcome.result };
|
|
3225
|
+
}
|
|
3226
|
+
this.lastPerf = measure;
|
|
3227
|
+
if (this.recording !== null && !this.replaying) {
|
|
3228
|
+
this.recording.steps.push({
|
|
3229
|
+
method: "performance",
|
|
3230
|
+
params: {
|
|
3231
|
+
...(args.launch === true ? { launch: true } : {}),
|
|
3232
|
+
...(args.seconds !== undefined ? { seconds: args.seconds } : {}),
|
|
3233
|
+
...(args.tap !== undefined ? { tap: args.tap } : {}),
|
|
3234
|
+
},
|
|
3235
|
+
});
|
|
3236
|
+
}
|
|
3237
|
+
const tapWords = args.tap === undefined ? undefined : targetWords(args.tap);
|
|
3238
|
+
return text([perfFirstLine(measure, tapWords), ...formatPerfLines(measure.result), ...this.takePersonNote()].join("\n"));
|
|
3239
|
+
}
|
|
3240
|
+
catch (error) {
|
|
3241
|
+
return fail(describeError(error));
|
|
3242
|
+
}
|
|
3243
|
+
}
|
|
2824
3244
|
async wait(args) {
|
|
2825
3245
|
const session = this.host.current();
|
|
2826
3246
|
if (session === undefined)
|