tapline-host 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actionlog.d.ts +28 -1
- package/dist/actionlog.js +2 -2
- package/dist/actionlog.js.map +1 -1
- package/dist/cli.js +10 -0
- package/dist/cli.js.map +1 -1
- package/dist/format.d.ts +2 -0
- package/dist/format.js +8 -0
- package/dist/format.js.map +1 -1
- package/dist/init.js +83 -1
- package/dist/init.js.map +1 -1
- package/dist/mcp.js +46 -32
- package/dist/mcp.js.map +1 -1
- package/dist/protocol.d.ts +2 -1
- package/dist/protocol.js +7 -1
- package/dist/protocol.js.map +1 -1
- package/dist/report.d.ts +55 -0
- package/dist/report.js +298 -0
- package/dist/report.js.map +1 -0
- package/dist/run.js +1 -1
- package/dist/run.js.map +1 -1
- package/dist/tool-text-data.d.ts +2 -1
- package/dist/tool-text-data.js +2 -1
- package/dist/tool-text-data.js.map +1 -1
- package/dist/tools.d.ts +49 -0
- package/dist/tools.js +276 -21
- package/dist/tools.js.map +1 -1
- package/package.json +1 -1
- package/skills/tapline/SKILL.md +33 -1
package/dist/tools.js
CHANGED
|
@@ -6,11 +6,13 @@ import { z } from "zod";
|
|
|
6
6
|
import { annotateScreenshot } from "./annotate.js";
|
|
7
7
|
import { agentKind, captionFor, personKind, reasonOf } from "./captions.js";
|
|
8
8
|
import { composeFilmstrip, diffImages, imageSize } from "./images.js";
|
|
9
|
-
import { alertNotes, describeElement, filterLogs, formatChanges, formatElement, formatLogAnswer, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
|
|
9
|
+
import { alertNotes, networkLine, describeElement, filterLogs, formatChanges, formatElement, formatLogAnswer, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
|
|
10
10
|
import { formatInspect, inspectResultSchema } from "./inspect.js";
|
|
11
11
|
import { AUDIT_TYPES, auditResultSchema, auditTitle, formatAudit } from "./audit.js";
|
|
12
12
|
import { NO_CRASH_TEXT, crashNote, disconnectNote, formatCrash } from "./crash.js";
|
|
13
13
|
import { ActionLog, loggedTarget } from "./actionlog.js";
|
|
14
|
+
import { reportFor } from "./report.js";
|
|
15
|
+
import { randomBytes } from "node:crypto";
|
|
14
16
|
import { Confirmations } from "./confirm.js";
|
|
15
17
|
import { DEFAULT_POLICY, elementKey, listsFor, matchWord } from "./policy.js";
|
|
16
18
|
import { readScript, refStep, scriptSteps } from "./script.js";
|
|
@@ -250,6 +252,9 @@ export const toolShapes = {
|
|
|
250
252
|
tap: targetField.optional().describe("Tap this first, then measure what it sets off"),
|
|
251
253
|
},
|
|
252
254
|
wait: { seconds: z.number().min(0.1).max(30).describe("How long to wait, 0.1 to 30 seconds"), screenshot: screenshotField },
|
|
255
|
+
session_report: {
|
|
256
|
+
since: z.number().min(0).optional().describe("Epoch milliseconds: report only calls from then on (default: the start of this session)"),
|
|
257
|
+
},
|
|
253
258
|
run: {
|
|
254
259
|
steps: z
|
|
255
260
|
.array(stepSchema)
|
|
@@ -632,7 +637,17 @@ const dryRunResultSchema = z.object({
|
|
|
632
637
|
dryRun: z.literal(true),
|
|
633
638
|
point: z.object({ x: z.number(), y: z.number() }).optional(),
|
|
634
639
|
value: z.unknown().optional(),
|
|
640
|
+
/** The element is off screen and the action would scroll it into view first (§16.2); `point` is where it expects to touch after. */
|
|
641
|
+
scrolls: z.boolean().optional(),
|
|
635
642
|
});
|
|
643
|
+
/** Whether an app's answer says the action scrolled the element into view first (AGENT-TOOLS 1.12 §16.2). */
|
|
644
|
+
function scrolledFirst(result) {
|
|
645
|
+
return typeof result === "object" && result !== null && "scrolled" in result && result.scrolled === true;
|
|
646
|
+
}
|
|
647
|
+
/** `Scrolled to it, then tapped …` when the action scrolled first: the sentence goes on after a lead-in. */
|
|
648
|
+
function afterScroll(result, sentence) {
|
|
649
|
+
return scrolledFirst(result) ? `Scrolled to it, then ${sentence.charAt(0).toLowerCase()}${sentence.slice(1)}` : sentence;
|
|
650
|
+
}
|
|
636
651
|
/** For an app without dry runs: every element on its list the target could mean. */
|
|
637
652
|
function candidatesFor(target, elements) {
|
|
638
653
|
if (typeof target !== "object" || target === null)
|
|
@@ -671,6 +686,34 @@ const CRASH_NOTED_TOOLS = [
|
|
|
671
686
|
"record", "replay", "tap", "longPress", "typeText", "scroll", "toggle", "appCommand", "waitFor", "swipe", "back",
|
|
672
687
|
"openUrl", "logs", "reload", "performance", "wait",
|
|
673
688
|
];
|
|
689
|
+
/** Every tool the agent calls, by handler name (the call log numbers these). */
|
|
690
|
+
const CALL_TOOLS = {
|
|
691
|
+
apps: "apps", openApp: "open_app", pair: "pair", look: "look", run: "run", inspect: "inspect", audit: "audit",
|
|
692
|
+
checkLayout: "check_layout", viewer: "viewer", compareScreen: "compare_screen", filmstrip: "filmstrip", record: "record",
|
|
693
|
+
replay: "replay", tap: "tap", longPress: "long_press", typeText: "type_text", scroll: "scroll", toggle: "toggle",
|
|
694
|
+
appCommand: "app_command", waitFor: "wait_for", swipe: "swipe", back: "back", openUrl: "open_url", logs: "logs",
|
|
695
|
+
crash: "crash", reload: "reload", performance: "performance", wait: "wait", sessionReport: "session_report",
|
|
696
|
+
};
|
|
697
|
+
/** Calls logged with `"kind": "read"` when they logged no action of their own. */
|
|
698
|
+
const READ_TOOLS = new Set([
|
|
699
|
+
"apps", "open_app", "pair", "look", "inspect", "audit", "check_layout", "viewer", "compare_screen", "filmstrip", "record",
|
|
700
|
+
"replay", "wait_for", "logs", "crash", "reload", "performance", "wait", "session_report",
|
|
701
|
+
]);
|
|
702
|
+
/** A look's picture arguments for the call log: booleans and numbers only; undefined when it gave none. */
|
|
703
|
+
function pictureArgs(input) {
|
|
704
|
+
const out = {};
|
|
705
|
+
if (typeof input["screenshot"] === "boolean")
|
|
706
|
+
out.screenshot = input["screenshot"];
|
|
707
|
+
if (typeof input["width"] === "number" && Number.isFinite(input["width"]))
|
|
708
|
+
out.width = input["width"];
|
|
709
|
+
if (typeof input["labels"] === "boolean")
|
|
710
|
+
out.labels = input["labels"];
|
|
711
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
712
|
+
}
|
|
713
|
+
/** Tools whose answer carries the changes or the full list unless the call said `look: false`. */
|
|
714
|
+
const LISTING_TOOLS = new Set(["tap", "long_press", "type_text", "scroll", "toggle", "swipe", "back", "open_url", "app_command", "run"]);
|
|
715
|
+
/** Tools that act on one element: after a run of scrolls they log whether the element was already in the list. */
|
|
716
|
+
const ELEMENT_ACTIONS = new Set(["tap", "long_press", "type_text", "toggle"]);
|
|
674
717
|
/** The tool handlers behind the MCP server. They only talk to a TaplineHost, so they are easy to test. */
|
|
675
718
|
export class TaplineTools {
|
|
676
719
|
host;
|
|
@@ -684,9 +727,28 @@ export class TaplineTools {
|
|
|
684
727
|
batch;
|
|
685
728
|
/** Extra fields for the next log line (a dry run, a confirmed touch). */
|
|
686
729
|
logExtra;
|
|
730
|
+
/** The call log (AGENT-TOOLS 1.11 §16.1): this run's id, when it began, and how many calls the agent has made. */
|
|
731
|
+
sessionTag = randomBytes(4).toString("hex");
|
|
732
|
+
startedAt = Date.now();
|
|
733
|
+
callCount = 0;
|
|
734
|
+
/** The call in flight (and everything it awaits): its log entries wait here for the size of its answer. */
|
|
735
|
+
callScope = new AsyncLocalStorage();
|
|
736
|
+
clientIdFn;
|
|
737
|
+
/** The tool of the previous call, for scroll runs. */
|
|
738
|
+
prevCallTool;
|
|
739
|
+
/**
|
|
740
|
+
* The latest full element list the agent was given. `stale`: only changes since. `unseen`: an action since then
|
|
741
|
+
* answered with no list and no changes (`look: false`), so the screen may be another one: neither "in the
|
|
742
|
+
* list" nor "not in it" can be said until a list is seen again.
|
|
743
|
+
*/
|
|
744
|
+
listBase;
|
|
745
|
+
/** The list as it was before the current run of scrolls began. */
|
|
746
|
+
scrollBase;
|
|
687
747
|
confirmations;
|
|
688
748
|
policy;
|
|
689
749
|
reloadSettleMs;
|
|
750
|
+
stillGapMs;
|
|
751
|
+
stillCapMs;
|
|
690
752
|
noScreenshot = new Set();
|
|
691
753
|
shotFiles = [];
|
|
692
754
|
/** Sessions this agent has had an element list for (so later lists can be just the changes). */
|
|
@@ -733,6 +795,8 @@ export class TaplineTools {
|
|
|
733
795
|
this.actionLog = options.logDir !== undefined ? new ActionLog(options.logDir) : null;
|
|
734
796
|
this.actionLog?.open();
|
|
735
797
|
this.reloadSettleMs = options.reloadSettleMs ?? 1500;
|
|
798
|
+
this.stillGapMs = options.stillGapMs ?? 150;
|
|
799
|
+
this.stillCapMs = options.stillCapMs ?? 2000;
|
|
736
800
|
// A finger on the phone itself: the app saw a real touch. It can't be queued like viewer input
|
|
737
801
|
// (it has already happened), so it only leaves the note and makes `{ref}` targets stale.
|
|
738
802
|
host.on("touch", (session) => {
|
|
@@ -748,6 +812,120 @@ export class TaplineTools {
|
|
|
748
812
|
const method = original;
|
|
749
813
|
Reflect.set(this, name, (...args) => this.noting(() => method.apply(this, args)));
|
|
750
814
|
}
|
|
815
|
+
// Outermost: every call of the agent is numbered and logged (reads too) once its answer is known.
|
|
816
|
+
for (const [name, tool] of Object.entries(CALL_TOOLS)) {
|
|
817
|
+
const original = Reflect.get(this, name);
|
|
818
|
+
if (typeof original !== "function")
|
|
819
|
+
continue;
|
|
820
|
+
const method = original;
|
|
821
|
+
Reflect.set(this, name, (...args) => this.loggedCall(tool, args[0], () => method.apply(this, args)));
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
/** The agent's client id for the call log; the MCP server passes the client's name once it is known. */
|
|
825
|
+
setClientId(read) {
|
|
826
|
+
this.clientIdFn = read;
|
|
827
|
+
}
|
|
828
|
+
/** A call refused before it reached its tool (an unknown argument): numbered and logged like any other, so the report counts it. */
|
|
829
|
+
rejectedCall(tool, args, answer) {
|
|
830
|
+
const result = this.loggedCall(tool, args, () => answer);
|
|
831
|
+
return result instanceof Promise ? answer : result;
|
|
832
|
+
}
|
|
833
|
+
/** Numbers a call of the agent and logs it when its answer is known (§16.1). Calls within a call (`run`'s steps) are part of it. */
|
|
834
|
+
loggedCall(tool, args, run) {
|
|
835
|
+
if (this.actionLog === null || this.callScope.getStore() !== undefined)
|
|
836
|
+
return run();
|
|
837
|
+
const afterScrolls = this.prevCallTool === "scroll";
|
|
838
|
+
if (tool === "scroll" && !afterScrolls)
|
|
839
|
+
this.scrollBase = this.listBase;
|
|
840
|
+
this.prevCallTool = tool;
|
|
841
|
+
const state = { call: ++this.callCount, tool, afterScrolls, entries: [] };
|
|
842
|
+
const began = new Date();
|
|
843
|
+
let result;
|
|
844
|
+
try {
|
|
845
|
+
result = this.callScope.run(state, run);
|
|
846
|
+
}
|
|
847
|
+
catch (error) {
|
|
848
|
+
this.flushCall(state, args, began, undefined);
|
|
849
|
+
throw error;
|
|
850
|
+
}
|
|
851
|
+
if (result instanceof Promise) {
|
|
852
|
+
return result.then((answer) => {
|
|
853
|
+
this.flushCall(state, args, began, answer);
|
|
854
|
+
return answer;
|
|
855
|
+
}, (error) => {
|
|
856
|
+
this.flushCall(state, args, began, undefined);
|
|
857
|
+
throw error;
|
|
858
|
+
});
|
|
859
|
+
}
|
|
860
|
+
this.flushCall(state, args, began, result);
|
|
861
|
+
return result;
|
|
862
|
+
}
|
|
863
|
+
flushCall(state, args, began, answer) {
|
|
864
|
+
if (this.actionLog === null)
|
|
865
|
+
return;
|
|
866
|
+
const said = (answer?.content ?? []).map((block) => (block.type === "text" ? block.text : ""));
|
|
867
|
+
const size = said.reduce((sum, part) => sum + part.length, 0);
|
|
868
|
+
const failed = answer === undefined || answer.isError === true;
|
|
869
|
+
const input = typeof args === "object" && args !== null ? args : {};
|
|
870
|
+
const listed = LISTING_TOOLS.has(state.tool) && !failed && input["look"] !== false && input["dry_run"] !== true && !(state.tool === "app_command" && input["name"] === undefined);
|
|
871
|
+
const client = (this.clientIdFn ?? this.options.clientId)?.()?.replace(/[^\w .@/-]/g, "").slice(0, 60);
|
|
872
|
+
const common = {
|
|
873
|
+
session: this.sessionTag,
|
|
874
|
+
call: state.call,
|
|
875
|
+
...(client !== undefined && client !== "" ? { client } : {}),
|
|
876
|
+
size,
|
|
877
|
+
};
|
|
878
|
+
const code = failed ? /^Error (\w+):/.exec(said.join("\n"))?.[1] : undefined;
|
|
879
|
+
if (state.entries.length > 0) {
|
|
880
|
+
// Actions that went through, in a call that failed after them (an expectation that didn't hold): the call's
|
|
881
|
+
// own outcome goes with them, so the report doesn't count it as a success (Codex review of 1.4.0).
|
|
882
|
+
const callFailed = failed && state.entries.every(({ entry }) => entry.ok) ? { call_failed: code ?? "internal" } : {};
|
|
883
|
+
for (const { entry, at } of state.entries)
|
|
884
|
+
this.actionLog.write({ ...entry, ...common, ...callFailed, ...(listed ? { listed: true } : {}) }, at);
|
|
885
|
+
return;
|
|
886
|
+
}
|
|
887
|
+
// A call that logged no action: a read, or an action refused before it began (then without `kind`).
|
|
888
|
+
const session = this.host.current();
|
|
889
|
+
const lookArgs = state.tool === "look" ? pictureArgs(input) : undefined;
|
|
890
|
+
this.actionLog.write({
|
|
891
|
+
by: "agent",
|
|
892
|
+
tool: state.tool,
|
|
893
|
+
...(READ_TOOLS.has(state.tool) ? { kind: "read" } : {}),
|
|
894
|
+
// the target only (as given); typed text, pictures and log text never
|
|
895
|
+
...(loggedTarget(args).target !== undefined ? { target: loggedTarget(args).target } : {}),
|
|
896
|
+
ok: !failed,
|
|
897
|
+
...(code !== undefined ? { code } : {}),
|
|
898
|
+
ms: Date.now() - began.getTime(),
|
|
899
|
+
...(session !== undefined ? { app: session.app.id ?? session.app.name } : {}),
|
|
900
|
+
...common,
|
|
901
|
+
...(listed ? { listed: true } : {}),
|
|
902
|
+
...(lookArgs !== undefined ? { look_args: lookArgs } : {}),
|
|
903
|
+
}, began);
|
|
904
|
+
}
|
|
905
|
+
/**
|
|
906
|
+
* Whether a touch's target was in the list the agent had before its run of scrolls: true (it was), false (it
|
|
907
|
+
* wasn't), undefined (can't be known: no list, a point, or a list that only changed since).
|
|
908
|
+
*/
|
|
909
|
+
knownBeforeScrolls(params) {
|
|
910
|
+
const base = this.scrollBase;
|
|
911
|
+
const target = typeof params === "object" && params !== null ? params.target : undefined;
|
|
912
|
+
if (base === undefined || typeof target !== "object" || target === null)
|
|
913
|
+
return undefined;
|
|
914
|
+
const t = target;
|
|
915
|
+
if (!["ref", "testID", "label", "text"].some((key) => typeof t[key] === "string"))
|
|
916
|
+
return undefined;
|
|
917
|
+
if (base.unseen)
|
|
918
|
+
return undefined;
|
|
919
|
+
if (candidatesFor(target, base.elements).length > 0)
|
|
920
|
+
return true;
|
|
921
|
+
return base.stale ? undefined : false;
|
|
922
|
+
}
|
|
923
|
+
/** `session_report` (AGENT-TOOLS 1.11 §16.1): where this session's calls went, and the waste in them. */
|
|
924
|
+
sessionReport(args = {}) {
|
|
925
|
+
const dir = this.options.logDir;
|
|
926
|
+
if (dir === undefined)
|
|
927
|
+
return fail("Error unsupported: the action log is off (--no-log), so there is nothing to report.");
|
|
928
|
+
return text(reportFor(dir, args.since ?? this.startedAt, this.sessionTag));
|
|
751
929
|
}
|
|
752
930
|
/**
|
|
753
931
|
* Runs a tool call and ends its answer with a note when the app stopped (AGENT-TOOLS 15.2): during this
|
|
@@ -1190,6 +1368,7 @@ export class TaplineTools {
|
|
|
1190
1368
|
let response;
|
|
1191
1369
|
const live = this.host.current();
|
|
1192
1370
|
if (args.look === false || live === undefined) {
|
|
1371
|
+
this.markListUnseen();
|
|
1193
1372
|
response = text([first, ...tail].join("\n"));
|
|
1194
1373
|
}
|
|
1195
1374
|
else {
|
|
@@ -1454,6 +1633,11 @@ export class TaplineTools {
|
|
|
1454
1633
|
else
|
|
1455
1634
|
this.alertOpen.delete(sid);
|
|
1456
1635
|
const changesText = askChanges ? formatChanges(tree, { alertGone }) : null;
|
|
1636
|
+
// For the call log: the list the agent now has (a change block leaves it as it was, but no longer certain).
|
|
1637
|
+
if (changesText === null)
|
|
1638
|
+
this.listBase = { elements: tree.elements, stale: false, unseen: false };
|
|
1639
|
+
else if (this.listBase !== undefined)
|
|
1640
|
+
this.listBase = { elements: this.listBase.elements, stale: true, unseen: false };
|
|
1457
1641
|
// A full list gives every ref a fresh meaning, so refs are trustworthy again.
|
|
1458
1642
|
if (changesText === null)
|
|
1459
1643
|
this.personActed = false;
|
|
@@ -1523,6 +1707,11 @@ export class TaplineTools {
|
|
|
1523
1707
|
this.personNotePending = false;
|
|
1524
1708
|
return ["Note: a person used the app since your last look."];
|
|
1525
1709
|
}
|
|
1710
|
+
/** An action answered with no list and no changes: the call log can't say what is in the list any more. */
|
|
1711
|
+
markListUnseen() {
|
|
1712
|
+
if (this.listBase !== undefined)
|
|
1713
|
+
this.listBase = { ...this.listBase, unseen: true };
|
|
1714
|
+
}
|
|
1526
1715
|
/** Runs a request, then (unless disabled) appends the fresh element list and how long the app took to settle. */
|
|
1527
1716
|
async act(method, params, describe, withLook, screenshot, extraNotes,
|
|
1528
1717
|
/** True when the result means nothing was done: the first line is then the whole answer. */
|
|
@@ -1570,6 +1759,7 @@ export class TaplineTools {
|
|
|
1570
1759
|
? `Settled in ${(settledMs / 1000).toFixed(2)} s.`
|
|
1571
1760
|
: undefined;
|
|
1572
1761
|
if (withLook === false) {
|
|
1762
|
+
this.markListUnseen();
|
|
1573
1763
|
this.unlistedActions.set(session.id, (this.unlistedActions.get(session.id) ?? 0) + 1);
|
|
1574
1764
|
response = text([line, ...this.takePersonNote()].join("\n"));
|
|
1575
1765
|
}
|
|
@@ -1688,7 +1878,8 @@ export class TaplineTools {
|
|
|
1688
1878
|
? elementSchema.safeParse(result.element)
|
|
1689
1879
|
: null;
|
|
1690
1880
|
const session = this.host.current();
|
|
1691
|
-
this.
|
|
1881
|
+
const state = by === "agent" ? this.callScope.getStore() : undefined;
|
|
1882
|
+
const entry = {
|
|
1692
1883
|
by,
|
|
1693
1884
|
tool,
|
|
1694
1885
|
...loggedTarget(params),
|
|
@@ -1701,9 +1892,22 @@ export class TaplineTools {
|
|
|
1701
1892
|
...(session !== undefined ? { app: session.app.id ?? session.app.name } : {}),
|
|
1702
1893
|
...(by === "agent" && this.batch !== undefined ? { batch: this.batch } : {}),
|
|
1703
1894
|
...(by === "agent" ? (this.logExtra ?? {}) : {}),
|
|
1704
|
-
}
|
|
1895
|
+
};
|
|
1705
1896
|
if (by === "agent")
|
|
1706
1897
|
this.logExtra = undefined;
|
|
1898
|
+
if (state === undefined) {
|
|
1899
|
+
this.actionLog.write(entry);
|
|
1900
|
+
return;
|
|
1901
|
+
}
|
|
1902
|
+
if (state.tool !== tool)
|
|
1903
|
+
entry.of = state.tool;
|
|
1904
|
+
// After scrolls: was the target already in the list before the first one? (a scroll hunt is exact when it was)
|
|
1905
|
+
if (state.afterScrolls && state.tool === tool && ELEMENT_ACTIONS.has(tool)) {
|
|
1906
|
+
const known = this.knownBeforeScrolls(params);
|
|
1907
|
+
if (known !== undefined)
|
|
1908
|
+
entry.in_list = known;
|
|
1909
|
+
}
|
|
1910
|
+
state.entries.push({ entry, at: new Date() });
|
|
1707
1911
|
}
|
|
1708
1912
|
showAction(by, method, result, params, code, reason) {
|
|
1709
1913
|
const onAction = this.options.onAction;
|
|
@@ -1959,7 +2163,7 @@ export class TaplineTools {
|
|
|
1959
2163
|
async checkAgainstReference(session, name, options) {
|
|
1960
2164
|
let shot;
|
|
1961
2165
|
try {
|
|
1962
|
-
shot = await this.
|
|
2166
|
+
shot = await this.stillShot(session.id, options.width, 0.9);
|
|
1963
2167
|
}
|
|
1964
2168
|
catch (error) {
|
|
1965
2169
|
return { kind: "error", line: describeError(error) };
|
|
@@ -2107,10 +2311,35 @@ export class TaplineTools {
|
|
|
2107
2311
|
// the failure still reads without the picture
|
|
2108
2312
|
}
|
|
2109
2313
|
}
|
|
2314
|
+
/**
|
|
2315
|
+
* A picture of the screen once it stopped moving: two in a row about 150 ms apart that agree (under 0.1% changed),
|
|
2316
|
+
* at most 2 s. The app's settle sees React commits, not a native transition, so a screen pushed by the action before
|
|
2317
|
+
* can still be sliding in (found on the phone, 2026-11-12). Past the cap, the last picture.
|
|
2318
|
+
*/
|
|
2319
|
+
async stillShot(sessionId, width, quality) {
|
|
2320
|
+
let previous = await this.takeShot(sessionId, width, quality);
|
|
2321
|
+
const started = Date.now();
|
|
2322
|
+
while (Date.now() - started < this.stillCapMs) {
|
|
2323
|
+
await sleep(this.stillGapMs);
|
|
2324
|
+
const next = await this.takeShot(sessionId, width, quality);
|
|
2325
|
+
if (next.data === previous.data)
|
|
2326
|
+
return next;
|
|
2327
|
+
try {
|
|
2328
|
+
const diff = diffImages(previous, next, { mask: [] });
|
|
2329
|
+
if (diff.changedPercent < 0.1)
|
|
2330
|
+
return next;
|
|
2331
|
+
}
|
|
2332
|
+
catch {
|
|
2333
|
+
return next; // sizes differ (a rotation): nothing to wait for
|
|
2334
|
+
}
|
|
2335
|
+
previous = next;
|
|
2336
|
+
}
|
|
2337
|
+
return previous;
|
|
2338
|
+
}
|
|
2110
2339
|
/** Takes the screen and saves it as the reference `name` in `dir`. */
|
|
2111
2340
|
async saveReferenceHere(session, name, width, dir) {
|
|
2112
2341
|
try {
|
|
2113
|
-
return this.writeReference(dir, name, await this.
|
|
2342
|
+
return this.writeReference(dir, name, await this.stillShot(session.id, width, 0.9));
|
|
2114
2343
|
}
|
|
2115
2344
|
catch (error) {
|
|
2116
2345
|
return fail(describeError(error));
|
|
@@ -2288,7 +2517,7 @@ export class TaplineTools {
|
|
|
2288
2517
|
const save = (shot) => this.writeReference(folder, name, shot);
|
|
2289
2518
|
if (args.update === true) {
|
|
2290
2519
|
try {
|
|
2291
|
-
return save(await this.
|
|
2520
|
+
return save(await this.stillShot(session.id, width, 0.9));
|
|
2292
2521
|
}
|
|
2293
2522
|
catch (error) {
|
|
2294
2523
|
return fail(describeError(error));
|
|
@@ -2611,7 +2840,7 @@ export class TaplineTools {
|
|
|
2611
2840
|
this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "blocked");
|
|
2612
2841
|
return fail(this.blockedLine(verdict.element, verdict.word));
|
|
2613
2842
|
}
|
|
2614
|
-
const would = this.wouldLine(tool, shown, resolved.point, resolved.value);
|
|
2843
|
+
const would = this.wouldLine(tool, shown, resolved.point, resolved.value, resolved.scrolls);
|
|
2615
2844
|
const needs = verdict.kind === "confirm" ? ` It needs a person's OK (policy word ${JSON.stringify(verdict.word)}).` : "";
|
|
2616
2845
|
this.showAction("agent", method, shown !== undefined ? { element: shown } : undefined, params);
|
|
2617
2846
|
return text([`${would}${needs}`, ...this.takePersonNote()].join("\n"));
|
|
@@ -2729,7 +2958,7 @@ export class TaplineTools {
|
|
|
2729
2958
|
return fail(resolved.text);
|
|
2730
2959
|
}
|
|
2731
2960
|
this.showAction("agent", method, resolved.elements[0] !== undefined ? { element: resolved.elements[0] } : undefined, params);
|
|
2732
|
-
return text([this.wouldLine(tool, resolved.elements[0], resolved.point, resolved.value), ...this.takePersonNote()].join("\n"));
|
|
2961
|
+
return text([this.wouldLine(tool, resolved.elements[0], resolved.point, resolved.value, resolved.scrolls), ...this.takePersonNote()].join("\n"));
|
|
2733
2962
|
}
|
|
2734
2963
|
/**
|
|
2735
2964
|
* What a touch would act on. Apps that know dry runs resolve it themselves, with every check; older
|
|
@@ -2758,6 +2987,7 @@ export class TaplineTools {
|
|
|
2758
2987
|
elements: parsed.data.element ? [parsed.data.element] : [],
|
|
2759
2988
|
...(parsed.data.point !== undefined ? { point: parsed.data.point } : {}),
|
|
2760
2989
|
...(parsed.data.value !== undefined ? { value: parsed.data.value } : {}),
|
|
2990
|
+
...(parsed.data.scrolls === true ? { scrolls: true } : {}),
|
|
2761
2991
|
};
|
|
2762
2992
|
}
|
|
2763
2993
|
// An app without dry runs: a swipe is checked by what lies under its start.
|
|
@@ -2788,24 +3018,27 @@ export class TaplineTools {
|
|
|
2788
3018
|
};
|
|
2789
3019
|
}
|
|
2790
3020
|
/** `Would tap e12 button "Delete trip" at 215,640.` */
|
|
2791
|
-
wouldLine(tool, element, point, value) {
|
|
2792
|
-
const name = element !== undefined ? this.shortName(element, true) : "the element";
|
|
3021
|
+
wouldLine(tool, element, point, value, scrolls) {
|
|
3022
|
+
const name = element !== undefined ? this.shortName(element, true, scrolls === true) : "the element";
|
|
3023
|
+
// Off screen (§16.2): `Would scroll to it, then tap e41 button "Save" id=save at 215,640.` The point is the one expected after the scroll.
|
|
3024
|
+
const lead = scrolls === true ? "Would scroll to it, then" : "Would";
|
|
2793
3025
|
const at = point !== undefined ? ` at ${point.x},${point.y}` : "";
|
|
2794
3026
|
switch (tool) {
|
|
2795
3027
|
case "long_press":
|
|
2796
|
-
return
|
|
3028
|
+
return `${lead} long-press ${name}${at}.`;
|
|
2797
3029
|
case "toggle":
|
|
2798
|
-
return
|
|
3030
|
+
return `${lead} toggle ${name}${typeof value === "boolean" ? ` to ${String(value)}` : ""}${at}.`;
|
|
2799
3031
|
case "type_text":
|
|
2800
|
-
return
|
|
3032
|
+
return `${lead} type into ${name}${scrolls === true ? at : ""}.`;
|
|
2801
3033
|
default:
|
|
2802
|
-
return
|
|
3034
|
+
return `${lead} tap ${name}${at}.`;
|
|
2803
3035
|
}
|
|
2804
3036
|
}
|
|
2805
3037
|
/** `e12 button "Delete trip"`: ref (when asked), role and name, nothing else. */
|
|
2806
|
-
shortName(element, withRef) {
|
|
3038
|
+
shortName(element, withRef, withId = false) {
|
|
2807
3039
|
return formatElement({
|
|
2808
3040
|
ref: withRef ? element.ref : "",
|
|
3041
|
+
...(withId ? { testID: element.testID ?? null } : {}),
|
|
2809
3042
|
role: element.role ?? null,
|
|
2810
3043
|
type: element.type ?? null,
|
|
2811
3044
|
label: element.label ?? null,
|
|
@@ -2827,12 +3060,12 @@ export class TaplineTools {
|
|
|
2827
3060
|
}
|
|
2828
3061
|
tap(args) {
|
|
2829
3062
|
const params = { target: toWireTarget(withIndex(args.target, args.index)), ...(args.force === true ? { force: true } : {}) };
|
|
2830
|
-
return this.guarded("tap", "press", params, args.dry_run === true, (preflight) => this.act("press", params, (r) => `Tapped ${this.elementName(r)}
|
|
3063
|
+
return this.guarded("tap", "press", params, args.dry_run === true, (preflight) => this.act("press", params, (r) => afterScroll(r, `Tapped ${this.elementName(r)}.`), args.look, args.screenshot, undefined, undefined, preflight));
|
|
2831
3064
|
}
|
|
2832
3065
|
/** `seconds` is accepted for the shared interface; Tapline calls the handler, so it doesn't hold. */
|
|
2833
3066
|
longPress(args) {
|
|
2834
3067
|
const params = { target: toWireTarget(withIndex(args.target, args.index)), ...(args.force === true ? { force: true } : {}) };
|
|
2835
|
-
return this.guarded("long_press", "longPress", params, args.dry_run === true, (preflight) => this.act("longPress", params, (r) => `Long-pressed ${this.elementName(r)}
|
|
3068
|
+
return this.guarded("long_press", "longPress", params, args.dry_run === true, (preflight) => this.act("longPress", params, (r) => afterScroll(r, `Long-pressed ${this.elementName(r)}.`), args.look, args.screenshot, undefined, undefined, preflight));
|
|
2836
3069
|
}
|
|
2837
3070
|
typeText(args) {
|
|
2838
3071
|
const params = {
|
|
@@ -2852,7 +3085,7 @@ export class TaplineTools {
|
|
|
2852
3085
|
// AGENT-TOOLS 1.8 §3.3: a secure field's value is never shown, not even as bullets in this line
|
|
2853
3086
|
const secure = typeof r === "object" && r !== null && "secure" in r && r.secure === true;
|
|
2854
3087
|
const shown = secure ? " The value is hidden (secure field)." : typeof value === "string" ? ` Value is now ${JSON.stringify(value)}.` : "";
|
|
2855
|
-
return `Typed into ${this.elementName(r, "element", shown !== "")}${args.submit ? " and submitted" : ""}.${shown}
|
|
3088
|
+
return afterScroll(r, `Typed into ${this.elementName(r, "element", shown !== "")}${args.submit ? " and submitted" : ""}.${shown}`);
|
|
2856
3089
|
}, args.look, args.screenshot);
|
|
2857
3090
|
}
|
|
2858
3091
|
scroll(args) {
|
|
@@ -2921,8 +3154,8 @@ export class TaplineTools {
|
|
|
2921
3154
|
return this.guarded("toggle", "toggle", params, args.dry_run === true, (preflight) => this.act("toggle", params, (r) => {
|
|
2922
3155
|
const value = typeof r === "object" && r !== null && "value" in r ? r.value : undefined;
|
|
2923
3156
|
if (unchanged(r))
|
|
2924
|
-
return `${this.elementName(r)} is already ${JSON.stringify(value)}.`;
|
|
2925
|
-
return `Toggled ${this.elementName(r)}${value !== undefined ? ` to ${JSON.stringify(value)}` : ""}
|
|
3157
|
+
return `${scrolledFirst(r) ? "Scrolled to it; " : ""}${this.elementName(r)} is already ${JSON.stringify(value)}.`;
|
|
3158
|
+
return afterScroll(r, `Toggled ${this.elementName(r)}${value !== undefined ? ` to ${JSON.stringify(value)}` : ""}.`);
|
|
2926
3159
|
}, args.look, args.screenshot, undefined, unchanged, preflight));
|
|
2927
3160
|
}
|
|
2928
3161
|
async appCommand(args) {
|
|
@@ -2946,8 +3179,28 @@ export class TaplineTools {
|
|
|
2946
3179
|
: `${session.app.name} offers no commands.`);
|
|
2947
3180
|
}
|
|
2948
3181
|
const name = args.name.trim();
|
|
3182
|
+
// AGENT-TOOLS 1.11 §16.4: a policy whose block list holds `network` refuses a condition; `normal` always goes.
|
|
3183
|
+
if (name === "network") {
|
|
3184
|
+
const mode = args.args?.mode;
|
|
3185
|
+
const app = this.host.current()?.app;
|
|
3186
|
+
const names = (words) => (words ?? []).some((word) => word.trim().toLowerCase() === "network");
|
|
3187
|
+
const own = app === undefined ? undefined : (this.policy.apps?.[app.id ?? ""] ?? this.policy.apps?.[app.name]);
|
|
3188
|
+
if (mode !== "normal" && names(listsFor(this.policy, app).block)) {
|
|
3189
|
+
const where = own?.block !== undefined && app !== undefined ? `for ${app.name}` : "here";
|
|
3190
|
+
return fail(`Error not_allowed: blocked: the safety policy blocks network ${where}.`);
|
|
3191
|
+
}
|
|
3192
|
+
}
|
|
2949
3193
|
return this.act("command", { name, args: args.args ?? {} }, (r) => {
|
|
2950
3194
|
const result = typeof r === "object" && r !== null && "result" in r ? r.result : null;
|
|
3195
|
+
// Built in `network`: the same line every header carries while a condition is on (AGENT-TOOLS §16.4),
|
|
3196
|
+
// then one sentence on what it reaches.
|
|
3197
|
+
if (name === "network" && typeof result === "object" && result !== null && "mode" in result && typeof result.mode === "string") {
|
|
3198
|
+
const covers = "covers" in result && typeof result.covers === "string" ? ` ${result.covers}` : "";
|
|
3199
|
+
return `${networkLine(result.mode)}${covers}`;
|
|
3200
|
+
}
|
|
3201
|
+
// Standard `language`, `region` and `text_size` (AGENT-TOOLS §16.3): the app's own sentence, e.g. `Text size xl (1.12×).`
|
|
3202
|
+
if (typeof result === "object" && result !== null && "say" in result && typeof result.say === "string")
|
|
3203
|
+
return result.say;
|
|
2951
3204
|
return result === null || result === undefined ? `Ran ${name}.` : `Ran ${name}: ${JSON.stringify(result)}`;
|
|
2952
3205
|
}, args.look, args.screenshot);
|
|
2953
3206
|
}
|
|
@@ -3067,8 +3320,10 @@ export class TaplineTools {
|
|
|
3067
3320
|
if (!worked)
|
|
3068
3321
|
return fail(describeError(error));
|
|
3069
3322
|
}
|
|
3070
|
-
if (args.look === false)
|
|
3323
|
+
if (args.look === false) {
|
|
3324
|
+
this.markListUnseen();
|
|
3071
3325
|
return text("Reload requested.");
|
|
3326
|
+
}
|
|
3072
3327
|
const deadline = Date.now() + this.reloadWaitMs;
|
|
3073
3328
|
for (;;) {
|
|
3074
3329
|
const now = this.host.current();
|