tapline-host 1.3.1 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/tools.js CHANGED
@@ -6,11 +6,13 @@ import { z } from "zod";
6
6
  import { annotateScreenshot } from "./annotate.js";
7
7
  import { agentKind, captionFor, personKind, reasonOf } from "./captions.js";
8
8
  import { composeFilmstrip, diffImages, imageSize } from "./images.js";
9
- import { alertNotes, describeElement, filterLogs, formatChanges, formatElement, formatLogAnswer, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
9
+ import { alertNotes, networkLine, describeElement, filterLogs, formatChanges, formatElement, formatLogAnswer, formatLogs, formatOffScreenAfterList, formatTree } from "./format.js";
10
10
  import { formatInspect, inspectResultSchema } from "./inspect.js";
11
11
  import { AUDIT_TYPES, auditResultSchema, auditTitle, formatAudit } from "./audit.js";
12
12
  import { NO_CRASH_TEXT, crashNote, disconnectNote, formatCrash } from "./crash.js";
13
13
  import { ActionLog, loggedTarget } from "./actionlog.js";
14
+ import { reportFor } from "./report.js";
15
+ import { randomBytes } from "node:crypto";
14
16
  import { Confirmations } from "./confirm.js";
15
17
  import { DEFAULT_POLICY, elementKey, listsFor, matchWord } from "./policy.js";
16
18
  import { readScript, refStep, scriptSteps } from "./script.js";
@@ -250,6 +252,9 @@ export const toolShapes = {
250
252
  tap: targetField.optional().describe("Tap this first, then measure what it sets off"),
251
253
  },
252
254
  wait: { seconds: z.number().min(0.1).max(30).describe("How long to wait, 0.1 to 30 seconds"), screenshot: screenshotField },
255
+ session_report: {
256
+ since: z.number().min(0).optional().describe("Epoch milliseconds: report only calls from then on (default: the start of this session)"),
257
+ },
253
258
  run: {
254
259
  steps: z
255
260
  .array(stepSchema)
@@ -632,7 +637,17 @@ const dryRunResultSchema = z.object({
632
637
  dryRun: z.literal(true),
633
638
  point: z.object({ x: z.number(), y: z.number() }).optional(),
634
639
  value: z.unknown().optional(),
640
+ /** The element is off screen and the action would scroll it into view first (§16.2); `point` is where it expects to touch after. */
641
+ scrolls: z.boolean().optional(),
635
642
  });
643
+ /** Whether an app's answer says the action scrolled the element into view first (AGENT-TOOLS 1.12 §16.2). */
644
+ function scrolledFirst(result) {
645
+ return typeof result === "object" && result !== null && "scrolled" in result && result.scrolled === true;
646
+ }
647
+ /** `Scrolled to it, then tapped …` when the action scrolled first: the sentence goes on after a lead-in. */
648
+ function afterScroll(result, sentence) {
649
+ return scrolledFirst(result) ? `Scrolled to it, then ${sentence.charAt(0).toLowerCase()}${sentence.slice(1)}` : sentence;
650
+ }
636
651
  /** For an app without dry runs: every element on its list the target could mean. */
637
652
  function candidatesFor(target, elements) {
638
653
  if (typeof target !== "object" || target === null)
@@ -671,6 +686,34 @@ const CRASH_NOTED_TOOLS = [
671
686
  "record", "replay", "tap", "longPress", "typeText", "scroll", "toggle", "appCommand", "waitFor", "swipe", "back",
672
687
  "openUrl", "logs", "reload", "performance", "wait",
673
688
  ];
689
+ /** Every tool the agent calls, by handler name (the call log numbers these). */
690
+ const CALL_TOOLS = {
691
+ apps: "apps", openApp: "open_app", pair: "pair", look: "look", run: "run", inspect: "inspect", audit: "audit",
692
+ checkLayout: "check_layout", viewer: "viewer", compareScreen: "compare_screen", filmstrip: "filmstrip", record: "record",
693
+ replay: "replay", tap: "tap", longPress: "long_press", typeText: "type_text", scroll: "scroll", toggle: "toggle",
694
+ appCommand: "app_command", waitFor: "wait_for", swipe: "swipe", back: "back", openUrl: "open_url", logs: "logs",
695
+ crash: "crash", reload: "reload", performance: "performance", wait: "wait", sessionReport: "session_report",
696
+ };
697
+ /** Calls logged with `"kind": "read"` when they logged no action of their own. */
698
+ const READ_TOOLS = new Set([
699
+ "apps", "open_app", "pair", "look", "inspect", "audit", "check_layout", "viewer", "compare_screen", "filmstrip", "record",
700
+ "replay", "wait_for", "logs", "crash", "reload", "performance", "wait", "session_report",
701
+ ]);
702
+ /** A look's picture arguments for the call log: booleans and numbers only; undefined when it gave none. */
703
+ function pictureArgs(input) {
704
+ const out = {};
705
+ if (typeof input["screenshot"] === "boolean")
706
+ out.screenshot = input["screenshot"];
707
+ if (typeof input["width"] === "number" && Number.isFinite(input["width"]))
708
+ out.width = input["width"];
709
+ if (typeof input["labels"] === "boolean")
710
+ out.labels = input["labels"];
711
+ return Object.keys(out).length > 0 ? out : undefined;
712
+ }
713
+ /** Tools whose answer carries the changes or the full list unless the call said `look: false`. */
714
+ const LISTING_TOOLS = new Set(["tap", "long_press", "type_text", "scroll", "toggle", "swipe", "back", "open_url", "app_command", "run"]);
715
+ /** Tools that act on one element: after a run of scrolls they log whether the element was already in the list. */
716
+ const ELEMENT_ACTIONS = new Set(["tap", "long_press", "type_text", "toggle"]);
674
717
  /** The tool handlers behind the MCP server. They only talk to a TaplineHost, so they are easy to test. */
675
718
  export class TaplineTools {
676
719
  host;
@@ -684,9 +727,28 @@ export class TaplineTools {
684
727
  batch;
685
728
  /** Extra fields for the next log line (a dry run, a confirmed touch). */
686
729
  logExtra;
730
+ /** The call log (AGENT-TOOLS 1.11 §16.1): this run's id, when it began, and how many calls the agent has made. */
731
+ sessionTag = randomBytes(4).toString("hex");
732
+ startedAt = Date.now();
733
+ callCount = 0;
734
+ /** The call in flight (and everything it awaits): its log entries wait here for the size of its answer. */
735
+ callScope = new AsyncLocalStorage();
736
+ clientIdFn;
737
+ /** The tool of the previous call, for scroll runs. */
738
+ prevCallTool;
739
+ /**
740
+ * The latest full element list the agent was given. `stale`: only changes since. `unseen`: an action since then
741
+ * answered with no list and no changes (`look: false`), so the screen may be another one: neither "in the
742
+ * list" nor "not in it" can be said until a list is seen again.
743
+ */
744
+ listBase;
745
+ /** The list as it was before the current run of scrolls began. */
746
+ scrollBase;
687
747
  confirmations;
688
748
  policy;
689
749
  reloadSettleMs;
750
+ stillGapMs;
751
+ stillCapMs;
690
752
  noScreenshot = new Set();
691
753
  shotFiles = [];
692
754
  /** Sessions this agent has had an element list for (so later lists can be just the changes). */
@@ -733,6 +795,8 @@ export class TaplineTools {
733
795
  this.actionLog = options.logDir !== undefined ? new ActionLog(options.logDir) : null;
734
796
  this.actionLog?.open();
735
797
  this.reloadSettleMs = options.reloadSettleMs ?? 1500;
798
+ this.stillGapMs = options.stillGapMs ?? 150;
799
+ this.stillCapMs = options.stillCapMs ?? 2000;
736
800
  // A finger on the phone itself: the app saw a real touch. It can't be queued like viewer input
737
801
  // (it has already happened), so it only leaves the note and makes `{ref}` targets stale.
738
802
  host.on("touch", (session) => {
@@ -748,6 +812,120 @@ export class TaplineTools {
748
812
  const method = original;
749
813
  Reflect.set(this, name, (...args) => this.noting(() => method.apply(this, args)));
750
814
  }
815
+ // Outermost: every call of the agent is numbered and logged (reads too) once its answer is known.
816
+ for (const [name, tool] of Object.entries(CALL_TOOLS)) {
817
+ const original = Reflect.get(this, name);
818
+ if (typeof original !== "function")
819
+ continue;
820
+ const method = original;
821
+ Reflect.set(this, name, (...args) => this.loggedCall(tool, args[0], () => method.apply(this, args)));
822
+ }
823
+ }
824
+ /** The agent's client id for the call log; the MCP server passes the client's name once it is known. */
825
+ setClientId(read) {
826
+ this.clientIdFn = read;
827
+ }
828
+ /** A call refused before it reached its tool (an unknown argument): numbered and logged like any other, so the report counts it. */
829
+ rejectedCall(tool, args, answer) {
830
+ const result = this.loggedCall(tool, args, () => answer);
831
+ return result instanceof Promise ? answer : result;
832
+ }
833
+ /** Numbers a call of the agent and logs it when its answer is known (§16.1). Calls within a call (`run`'s steps) are part of it. */
834
+ loggedCall(tool, args, run) {
835
+ if (this.actionLog === null || this.callScope.getStore() !== undefined)
836
+ return run();
837
+ const afterScrolls = this.prevCallTool === "scroll";
838
+ if (tool === "scroll" && !afterScrolls)
839
+ this.scrollBase = this.listBase;
840
+ this.prevCallTool = tool;
841
+ const state = { call: ++this.callCount, tool, afterScrolls, entries: [] };
842
+ const began = new Date();
843
+ let result;
844
+ try {
845
+ result = this.callScope.run(state, run);
846
+ }
847
+ catch (error) {
848
+ this.flushCall(state, args, began, undefined);
849
+ throw error;
850
+ }
851
+ if (result instanceof Promise) {
852
+ return result.then((answer) => {
853
+ this.flushCall(state, args, began, answer);
854
+ return answer;
855
+ }, (error) => {
856
+ this.flushCall(state, args, began, undefined);
857
+ throw error;
858
+ });
859
+ }
860
+ this.flushCall(state, args, began, result);
861
+ return result;
862
+ }
863
+ flushCall(state, args, began, answer) {
864
+ if (this.actionLog === null)
865
+ return;
866
+ const said = (answer?.content ?? []).map((block) => (block.type === "text" ? block.text : ""));
867
+ const size = said.reduce((sum, part) => sum + part.length, 0);
868
+ const failed = answer === undefined || answer.isError === true;
869
+ const input = typeof args === "object" && args !== null ? args : {};
870
+ const listed = LISTING_TOOLS.has(state.tool) && !failed && input["look"] !== false && input["dry_run"] !== true && !(state.tool === "app_command" && input["name"] === undefined);
871
+ const client = (this.clientIdFn ?? this.options.clientId)?.()?.replace(/[^\w .@/-]/g, "").slice(0, 60);
872
+ const common = {
873
+ session: this.sessionTag,
874
+ call: state.call,
875
+ ...(client !== undefined && client !== "" ? { client } : {}),
876
+ size,
877
+ };
878
+ const code = failed ? /^Error (\w+):/.exec(said.join("\n"))?.[1] : undefined;
879
+ if (state.entries.length > 0) {
880
+ // Actions that went through, in a call that failed after them (an expectation that didn't hold): the call's
881
+ // own outcome goes with them, so the report doesn't count it as a success (Codex review of 1.4.0).
882
+ const callFailed = failed && state.entries.every(({ entry }) => entry.ok) ? { call_failed: code ?? "internal" } : {};
883
+ for (const { entry, at } of state.entries)
884
+ this.actionLog.write({ ...entry, ...common, ...callFailed, ...(listed ? { listed: true } : {}) }, at);
885
+ return;
886
+ }
887
+ // A call that logged no action: a read, or an action refused before it began (then without `kind`).
888
+ const session = this.host.current();
889
+ const lookArgs = state.tool === "look" ? pictureArgs(input) : undefined;
890
+ this.actionLog.write({
891
+ by: "agent",
892
+ tool: state.tool,
893
+ ...(READ_TOOLS.has(state.tool) ? { kind: "read" } : {}),
894
+ // the target only (as given); typed text, pictures and log text never
895
+ ...(loggedTarget(args).target !== undefined ? { target: loggedTarget(args).target } : {}),
896
+ ok: !failed,
897
+ ...(code !== undefined ? { code } : {}),
898
+ ms: Date.now() - began.getTime(),
899
+ ...(session !== undefined ? { app: session.app.id ?? session.app.name } : {}),
900
+ ...common,
901
+ ...(listed ? { listed: true } : {}),
902
+ ...(lookArgs !== undefined ? { look_args: lookArgs } : {}),
903
+ }, began);
904
+ }
905
+ /**
906
+ * Whether a touch's target was in the list the agent had before its run of scrolls: true (it was), false (it
907
+ * wasn't), undefined (can't be known: no list, a point, or a list that only changed since).
908
+ */
909
+ knownBeforeScrolls(params) {
910
+ const base = this.scrollBase;
911
+ const target = typeof params === "object" && params !== null ? params.target : undefined;
912
+ if (base === undefined || typeof target !== "object" || target === null)
913
+ return undefined;
914
+ const t = target;
915
+ if (!["ref", "testID", "label", "text"].some((key) => typeof t[key] === "string"))
916
+ return undefined;
917
+ if (base.unseen)
918
+ return undefined;
919
+ if (candidatesFor(target, base.elements).length > 0)
920
+ return true;
921
+ return base.stale ? undefined : false;
922
+ }
923
+ /** `session_report` (AGENT-TOOLS 1.11 §16.1): where this session's calls went, and the waste in them. */
924
+ sessionReport(args = {}) {
925
+ const dir = this.options.logDir;
926
+ if (dir === undefined)
927
+ return fail("Error unsupported: the action log is off (--no-log), so there is nothing to report.");
928
+ return text(reportFor(dir, args.since ?? this.startedAt, this.sessionTag));
751
929
  }
752
930
  /**
753
931
  * Runs a tool call and ends its answer with a note when the app stopped (AGENT-TOOLS 15.2): during this
@@ -1190,6 +1368,7 @@ export class TaplineTools {
1190
1368
  let response;
1191
1369
  const live = this.host.current();
1192
1370
  if (args.look === false || live === undefined) {
1371
+ this.markListUnseen();
1193
1372
  response = text([first, ...tail].join("\n"));
1194
1373
  }
1195
1374
  else {
@@ -1454,6 +1633,11 @@ export class TaplineTools {
1454
1633
  else
1455
1634
  this.alertOpen.delete(sid);
1456
1635
  const changesText = askChanges ? formatChanges(tree, { alertGone }) : null;
1636
+ // For the call log: the list the agent now has (a change block leaves it as it was, but no longer certain).
1637
+ if (changesText === null)
1638
+ this.listBase = { elements: tree.elements, stale: false, unseen: false };
1639
+ else if (this.listBase !== undefined)
1640
+ this.listBase = { elements: this.listBase.elements, stale: true, unseen: false };
1457
1641
  // A full list gives every ref a fresh meaning, so refs are trustworthy again.
1458
1642
  if (changesText === null)
1459
1643
  this.personActed = false;
@@ -1523,6 +1707,11 @@ export class TaplineTools {
1523
1707
  this.personNotePending = false;
1524
1708
  return ["Note: a person used the app since your last look."];
1525
1709
  }
1710
+ /** An action answered with no list and no changes: the call log can't say what is in the list any more. */
1711
+ markListUnseen() {
1712
+ if (this.listBase !== undefined)
1713
+ this.listBase = { ...this.listBase, unseen: true };
1714
+ }
1526
1715
  /** Runs a request, then (unless disabled) appends the fresh element list and how long the app took to settle. */
1527
1716
  async act(method, params, describe, withLook, screenshot, extraNotes,
1528
1717
  /** True when the result means nothing was done: the first line is then the whole answer. */
@@ -1570,6 +1759,7 @@ export class TaplineTools {
1570
1759
  ? `Settled in ${(settledMs / 1000).toFixed(2)} s.`
1571
1760
  : undefined;
1572
1761
  if (withLook === false) {
1762
+ this.markListUnseen();
1573
1763
  this.unlistedActions.set(session.id, (this.unlistedActions.get(session.id) ?? 0) + 1);
1574
1764
  response = text([line, ...this.takePersonNote()].join("\n"));
1575
1765
  }
@@ -1688,7 +1878,8 @@ export class TaplineTools {
1688
1878
  ? elementSchema.safeParse(result.element)
1689
1879
  : null;
1690
1880
  const session = this.host.current();
1691
- this.actionLog.write({
1881
+ const state = by === "agent" ? this.callScope.getStore() : undefined;
1882
+ const entry = {
1692
1883
  by,
1693
1884
  tool,
1694
1885
  ...loggedTarget(params),
@@ -1701,9 +1892,22 @@ export class TaplineTools {
1701
1892
  ...(session !== undefined ? { app: session.app.id ?? session.app.name } : {}),
1702
1893
  ...(by === "agent" && this.batch !== undefined ? { batch: this.batch } : {}),
1703
1894
  ...(by === "agent" ? (this.logExtra ?? {}) : {}),
1704
- });
1895
+ };
1705
1896
  if (by === "agent")
1706
1897
  this.logExtra = undefined;
1898
+ if (state === undefined) {
1899
+ this.actionLog.write(entry);
1900
+ return;
1901
+ }
1902
+ if (state.tool !== tool)
1903
+ entry.of = state.tool;
1904
+ // After scrolls: was the target already in the list before the first one? (a scroll hunt is exact when it was)
1905
+ if (state.afterScrolls && state.tool === tool && ELEMENT_ACTIONS.has(tool)) {
1906
+ const known = this.knownBeforeScrolls(params);
1907
+ if (known !== undefined)
1908
+ entry.in_list = known;
1909
+ }
1910
+ state.entries.push({ entry, at: new Date() });
1707
1911
  }
1708
1912
  showAction(by, method, result, params, code, reason) {
1709
1913
  const onAction = this.options.onAction;
@@ -1959,7 +2163,7 @@ export class TaplineTools {
1959
2163
  async checkAgainstReference(session, name, options) {
1960
2164
  let shot;
1961
2165
  try {
1962
- shot = await this.takeShot(session.id, options.width, 0.9);
2166
+ shot = await this.stillShot(session.id, options.width, 0.9);
1963
2167
  }
1964
2168
  catch (error) {
1965
2169
  return { kind: "error", line: describeError(error) };
@@ -2107,10 +2311,35 @@ export class TaplineTools {
2107
2311
  // the failure still reads without the picture
2108
2312
  }
2109
2313
  }
2314
+ /**
2315
+ * A picture of the screen once it stopped moving: two in a row about 150 ms apart that agree (under 0.1% changed),
2316
+ * at most 2 s. The app's settle sees React commits, not a native transition, so a screen pushed by the action before
2317
+ * can still be sliding in (found on the phone, 2026-11-12). Past the cap, the last picture.
2318
+ */
2319
+ async stillShot(sessionId, width, quality) {
2320
+ let previous = await this.takeShot(sessionId, width, quality);
2321
+ const started = Date.now();
2322
+ while (Date.now() - started < this.stillCapMs) {
2323
+ await sleep(this.stillGapMs);
2324
+ const next = await this.takeShot(sessionId, width, quality);
2325
+ if (next.data === previous.data)
2326
+ return next;
2327
+ try {
2328
+ const diff = diffImages(previous, next, { mask: [] });
2329
+ if (diff.changedPercent < 0.1)
2330
+ return next;
2331
+ }
2332
+ catch {
2333
+ return next; // sizes differ (a rotation): nothing to wait for
2334
+ }
2335
+ previous = next;
2336
+ }
2337
+ return previous;
2338
+ }
2110
2339
  /** Takes the screen and saves it as the reference `name` in `dir`. */
2111
2340
  async saveReferenceHere(session, name, width, dir) {
2112
2341
  try {
2113
- return this.writeReference(dir, name, await this.takeShot(session.id, width, 0.9));
2342
+ return this.writeReference(dir, name, await this.stillShot(session.id, width, 0.9));
2114
2343
  }
2115
2344
  catch (error) {
2116
2345
  return fail(describeError(error));
@@ -2288,7 +2517,7 @@ export class TaplineTools {
2288
2517
  const save = (shot) => this.writeReference(folder, name, shot);
2289
2518
  if (args.update === true) {
2290
2519
  try {
2291
- return save(await this.takeShot(session.id, width, 0.9));
2520
+ return save(await this.stillShot(session.id, width, 0.9));
2292
2521
  }
2293
2522
  catch (error) {
2294
2523
  return fail(describeError(error));
@@ -2611,7 +2840,7 @@ export class TaplineTools {
2611
2840
  this.showAction("agent", method, { element: verdict.element }, params, "not_allowed", "blocked");
2612
2841
  return fail(this.blockedLine(verdict.element, verdict.word));
2613
2842
  }
2614
- const would = this.wouldLine(tool, shown, resolved.point, resolved.value);
2843
+ const would = this.wouldLine(tool, shown, resolved.point, resolved.value, resolved.scrolls);
2615
2844
  const needs = verdict.kind === "confirm" ? ` It needs a person's OK (policy word ${JSON.stringify(verdict.word)}).` : "";
2616
2845
  this.showAction("agent", method, shown !== undefined ? { element: shown } : undefined, params);
2617
2846
  return text([`${would}${needs}`, ...this.takePersonNote()].join("\n"));
@@ -2729,7 +2958,7 @@ export class TaplineTools {
2729
2958
  return fail(resolved.text);
2730
2959
  }
2731
2960
  this.showAction("agent", method, resolved.elements[0] !== undefined ? { element: resolved.elements[0] } : undefined, params);
2732
- return text([this.wouldLine(tool, resolved.elements[0], resolved.point, resolved.value), ...this.takePersonNote()].join("\n"));
2961
+ return text([this.wouldLine(tool, resolved.elements[0], resolved.point, resolved.value, resolved.scrolls), ...this.takePersonNote()].join("\n"));
2733
2962
  }
2734
2963
  /**
2735
2964
  * What a touch would act on. Apps that know dry runs resolve it themselves, with every check; older
@@ -2758,6 +2987,7 @@ export class TaplineTools {
2758
2987
  elements: parsed.data.element ? [parsed.data.element] : [],
2759
2988
  ...(parsed.data.point !== undefined ? { point: parsed.data.point } : {}),
2760
2989
  ...(parsed.data.value !== undefined ? { value: parsed.data.value } : {}),
2990
+ ...(parsed.data.scrolls === true ? { scrolls: true } : {}),
2761
2991
  };
2762
2992
  }
2763
2993
  // An app without dry runs: a swipe is checked by what lies under its start.
@@ -2788,24 +3018,27 @@ export class TaplineTools {
2788
3018
  };
2789
3019
  }
2790
3020
  /** `Would tap e12 button "Delete trip" at 215,640.` */
2791
- wouldLine(tool, element, point, value) {
2792
- const name = element !== undefined ? this.shortName(element, true) : "the element";
3021
+ wouldLine(tool, element, point, value, scrolls) {
3022
+ const name = element !== undefined ? this.shortName(element, true, scrolls === true) : "the element";
3023
+ // Off screen (§16.2): `Would scroll to it, then tap e41 button "Save" id=save at 215,640.` The point is the one expected after the scroll.
3024
+ const lead = scrolls === true ? "Would scroll to it, then" : "Would";
2793
3025
  const at = point !== undefined ? ` at ${point.x},${point.y}` : "";
2794
3026
  switch (tool) {
2795
3027
  case "long_press":
2796
- return `Would long-press ${name}${at}.`;
3028
+ return `${lead} long-press ${name}${at}.`;
2797
3029
  case "toggle":
2798
- return `Would toggle ${name}${typeof value === "boolean" ? ` to ${String(value)}` : ""}${at}.`;
3030
+ return `${lead} toggle ${name}${typeof value === "boolean" ? ` to ${String(value)}` : ""}${at}.`;
2799
3031
  case "type_text":
2800
- return `Would type into ${name}.`;
3032
+ return `${lead} type into ${name}${scrolls === true ? at : ""}.`;
2801
3033
  default:
2802
- return `Would tap ${name}${at}.`;
3034
+ return `${lead} tap ${name}${at}.`;
2803
3035
  }
2804
3036
  }
2805
3037
  /** `e12 button "Delete trip"`: ref (when asked), role and name, nothing else. */
2806
- shortName(element, withRef) {
3038
+ shortName(element, withRef, withId = false) {
2807
3039
  return formatElement({
2808
3040
  ref: withRef ? element.ref : "",
3041
+ ...(withId ? { testID: element.testID ?? null } : {}),
2809
3042
  role: element.role ?? null,
2810
3043
  type: element.type ?? null,
2811
3044
  label: element.label ?? null,
@@ -2827,12 +3060,12 @@ export class TaplineTools {
2827
3060
  }
2828
3061
  tap(args) {
2829
3062
  const params = { target: toWireTarget(withIndex(args.target, args.index)), ...(args.force === true ? { force: true } : {}) };
2830
- return this.guarded("tap", "press", params, args.dry_run === true, (preflight) => this.act("press", params, (r) => `Tapped ${this.elementName(r)}.`, args.look, args.screenshot, undefined, undefined, preflight));
3063
+ return this.guarded("tap", "press", params, args.dry_run === true, (preflight) => this.act("press", params, (r) => afterScroll(r, `Tapped ${this.elementName(r)}.`), args.look, args.screenshot, undefined, undefined, preflight));
2831
3064
  }
2832
3065
  /** `seconds` is accepted for the shared interface; Tapline calls the handler, so it doesn't hold. */
2833
3066
  longPress(args) {
2834
3067
  const params = { target: toWireTarget(withIndex(args.target, args.index)), ...(args.force === true ? { force: true } : {}) };
2835
- return this.guarded("long_press", "longPress", params, args.dry_run === true, (preflight) => this.act("longPress", params, (r) => `Long-pressed ${this.elementName(r)}.`, args.look, args.screenshot, undefined, undefined, preflight));
3068
+ return this.guarded("long_press", "longPress", params, args.dry_run === true, (preflight) => this.act("longPress", params, (r) => afterScroll(r, `Long-pressed ${this.elementName(r)}.`), args.look, args.screenshot, undefined, undefined, preflight));
2836
3069
  }
2837
3070
  typeText(args) {
2838
3071
  const params = {
@@ -2852,7 +3085,7 @@ export class TaplineTools {
2852
3085
  // AGENT-TOOLS 1.8 §3.3: a secure field's value is never shown, not even as bullets in this line
2853
3086
  const secure = typeof r === "object" && r !== null && "secure" in r && r.secure === true;
2854
3087
  const shown = secure ? " The value is hidden (secure field)." : typeof value === "string" ? ` Value is now ${JSON.stringify(value)}.` : "";
2855
- return `Typed into ${this.elementName(r, "element", shown !== "")}${args.submit ? " and submitted" : ""}.${shown}`;
3088
+ return afterScroll(r, `Typed into ${this.elementName(r, "element", shown !== "")}${args.submit ? " and submitted" : ""}.${shown}`);
2856
3089
  }, args.look, args.screenshot);
2857
3090
  }
2858
3091
  scroll(args) {
@@ -2921,8 +3154,8 @@ export class TaplineTools {
2921
3154
  return this.guarded("toggle", "toggle", params, args.dry_run === true, (preflight) => this.act("toggle", params, (r) => {
2922
3155
  const value = typeof r === "object" && r !== null && "value" in r ? r.value : undefined;
2923
3156
  if (unchanged(r))
2924
- return `${this.elementName(r)} is already ${JSON.stringify(value)}.`;
2925
- return `Toggled ${this.elementName(r)}${value !== undefined ? ` to ${JSON.stringify(value)}` : ""}.`;
3157
+ return `${scrolledFirst(r) ? "Scrolled to it; " : ""}${this.elementName(r)} is already ${JSON.stringify(value)}.`;
3158
+ return afterScroll(r, `Toggled ${this.elementName(r)}${value !== undefined ? ` to ${JSON.stringify(value)}` : ""}.`);
2926
3159
  }, args.look, args.screenshot, undefined, unchanged, preflight));
2927
3160
  }
2928
3161
  async appCommand(args) {
@@ -2946,8 +3179,28 @@ export class TaplineTools {
2946
3179
  : `${session.app.name} offers no commands.`);
2947
3180
  }
2948
3181
  const name = args.name.trim();
3182
+ // AGENT-TOOLS 1.11 §16.4: a policy whose block list holds `network` refuses a condition; `normal` always goes.
3183
+ if (name === "network") {
3184
+ const mode = args.args?.mode;
3185
+ const app = this.host.current()?.app;
3186
+ const names = (words) => (words ?? []).some((word) => word.trim().toLowerCase() === "network");
3187
+ const own = app === undefined ? undefined : (this.policy.apps?.[app.id ?? ""] ?? this.policy.apps?.[app.name]);
3188
+ if (mode !== "normal" && names(listsFor(this.policy, app).block)) {
3189
+ const where = own?.block !== undefined && app !== undefined ? `for ${app.name}` : "here";
3190
+ return fail(`Error not_allowed: blocked: the safety policy blocks network ${where}.`);
3191
+ }
3192
+ }
2949
3193
  return this.act("command", { name, args: args.args ?? {} }, (r) => {
2950
3194
  const result = typeof r === "object" && r !== null && "result" in r ? r.result : null;
3195
+ // Built in `network`: the same line every header carries while a condition is on (AGENT-TOOLS §16.4),
3196
+ // then one sentence on what it reaches.
3197
+ if (name === "network" && typeof result === "object" && result !== null && "mode" in result && typeof result.mode === "string") {
3198
+ const covers = "covers" in result && typeof result.covers === "string" ? ` ${result.covers}` : "";
3199
+ return `${networkLine(result.mode)}${covers}`;
3200
+ }
3201
+ // Standard `language`, `region` and `text_size` (AGENT-TOOLS §16.3): the app's own sentence, e.g. `Text size xl (1.12×).`
3202
+ if (typeof result === "object" && result !== null && "say" in result && typeof result.say === "string")
3203
+ return result.say;
2951
3204
  return result === null || result === undefined ? `Ran ${name}.` : `Ran ${name}: ${JSON.stringify(result)}`;
2952
3205
  }, args.look, args.screenshot);
2953
3206
  }
@@ -3067,8 +3320,10 @@ export class TaplineTools {
3067
3320
  if (!worked)
3068
3321
  return fail(describeError(error));
3069
3322
  }
3070
- if (args.look === false)
3323
+ if (args.look === false) {
3324
+ this.markListUnseen();
3071
3325
  return text("Reload requested.");
3326
+ }
3072
3327
  const deadline = Date.now() + this.reloadWaitMs;
3073
3328
  for (;;) {
3074
3329
  const now = this.host.current();