touchpress 0.0.2 → 0.1.0

package/README.md

@@ -66,6 +66,19 @@ npx playwright test --project=ios
 
 `preflight` reads a project's options and reports whether the device they name is booted, so a missing simulator fails once, in about a second, instead of once per test after the launch timeout. Wire it as a setup project per platform that the device projects depend on. [Basics](https://github.com/wobsoriano/touchpress/blob/main/docs/basics.md) has the spec, and [`apps/e2e/e2e/preflight.setup.mts`](https://github.com/wobsoriano/touchpress/blob/main/apps/e2e/e2e/preflight.setup.mts) is a working one.
 
+### Drive a step with a model
+
+`device.act` takes an instruction in English and drives the app until it is satisfied. `device.extract` asks one question about the screen and returns a typed answer.
+
+```ts
+test('sign in', async ({ device }) => {
+  await device.act('Sign in with the email rob@example.com and the password hunter2');
+  await expect(device.getByTestId('greeting')).toHaveText('Hi, Rob');
+});
+```
+
+Set `use.aiModel` to a gateway model id or a provider model instance, and install `ai`, an optional peer dependency. A loop can run for minutes, so raise the test timeout on any spec that calls `act`. The example uses the sample app's fake account. Keep real credentials in deterministic `fill(text, { secret: true })` calls outside `act`, and assert deterministically afterwards. [AI](https://github.com/wobsoriano/touchpress/blob/main/docs/ai.md) covers `act`, `extract`, the tools the model gets, and the report.
+
 ## Run the sample project
 
 `apps/e2e` is an Expo app with a home, login, and profile route and a fake sign-in. It is the app touchpress is tested against. Build the library first with `vp run -r build` so the app can resolve `dist`, then run these from `apps/e2e`:
@@ -93,6 +106,7 @@ Leave Metro running for the whole suite. The first build takes several minutes.
 - [Configuration](https://github.com/wobsoriano/touchpress/blob/main/docs/configuration.md)
 - [Locators](https://github.com/wobsoriano/touchpress/blob/main/docs/locators.md)
 - [Assertions](https://github.com/wobsoriano/touchpress/blob/main/docs/assertions.md)
+- [AI](https://github.com/wobsoriano/touchpress/blob/main/docs/ai.md)
 - [Lifecycle](https://github.com/wobsoriano/touchpress/blob/main/docs/lifecycle.md)
 - [Continuous integration](https://github.com/wobsoriano/touchpress/blob/main/docs/ci.md)
 

package/dist/core/index.d.mts

@@ -1,4 +1,4 @@
-import { $ as Tree, A as deviceNameForSlot, B as ProbeResult, C as openSession, D as ResolvedOptions, E as ReadyQuery, F as StepOptions, G as CaptureOptions, H as formatFailure, I as Typed, J as DeviceInfo, K as DeviceDriver, L as renderTitle, M as ActionRecord, N as ActionSink, O as TOUCHPRESS_DEFAULTS, P as EvidenceFile, Q as Settled, R as silentSink, S as SessionState, T as DeviceChoice, U as probe, V as ProbeTarget, W as Binding, X as OpenRequest, Y as DeviceSelection, Z as ScrollDirection, _ as RoleOptions, _t as Screen, a as ExpectedValue, at as Filter, b as DeviceSession, bt as renderScreen, c as ScrollSearch, ct as TextMatch, d as directionToward, dt as textMatch, et as Check, f as ActionOptions, ft as PinnedRef, g as Locator, gt as Resolution, h as FilterOptions, ht as Rect, i as ErrorInfo, it as evaluate, j as parseDeviceOptions, k as TouchpressOptions, l as ScrollTrail, lt as describeQuery, m as FillOptions, mt as RawSnapshot, n as PreflightReport, nt as Verdict, o as TouchpressError, ot as Query, p as Device, pt as Platform, q as DeviceFailure, r as preflight, rt as describeCheck, s as ScrollDevice, st as Role, t as PreflightDevice, tt as CheckName, u as createScrollSearch, ut as normalizeText, v as TextOptions, vt as ScreenNode, w as sessionName, x as OpenSessionInput, xt as resolve, y as createDevice, yt as parseScreen, z as ProbeOptions } from "../preflight-CCKYplYi.mjs";
+import { $ as Tree, A as deviceNameForSlot, B as ProbeResult, C as openSession, D as ResolvedOptions, E as ReadyQuery, F as StepOptions, G as CaptureOptions, H as formatFailure, I as Typed, J as DeviceInfo, K as DeviceDriver, L as renderTitle, M as ActionRecord, N as ActionSink, O as TOUCHPRESS_DEFAULTS, P as EvidenceFile, Q as Settled, R as silentSink, S as SessionState, T as DeviceChoice, U as probe, V as ProbeTarget, W as Binding, X as OpenRequest, Y as DeviceSelection, Z as ScrollDirection, _ as RoleOptions, _t as Screen, a as ExpectedValue, at as Filter, b as DeviceSession, bt as renderScreen, c as ScrollSearch, ct as TextMatch, d as directionToward, dt as textMatch, et as Check, f as ActionOptions, ft as PinnedRef, g as Locator, gt as Resolution, h as FilterOptions, ht as Rect, i as ErrorInfo, it as evaluate, j as parseDeviceOptions, k as TouchpressOptions, l as ScrollTrail, lt as describeQuery, m as FillOptions, mt as RawSnapshot, n as PreflightReport, nt as Verdict, o as TouchpressError, ot as Query, p as Device, pt as Platform, q as DeviceFailure, r as preflight, rt as describeCheck, s as ScrollDevice, st as Role, t as PreflightDevice, tt as CheckName, u as createScrollSearch, ut as normalizeText, v as TextOptions, vt as ScreenNode, w as sessionName, x as OpenSessionInput, xt as resolve, y as createDevice, yt as parseScreen, z as ProbeOptions } from "../preflight-Czhj9QFM.mjs";
 //#region src/core/evidence.d.ts
 /**
  * Never throws. A capture that fails records a note and returns, because masking

package/dist/core/index.mjs

@@ -1,2 +1,2 @@
-import { A as textMatch, C as describeCheck, D as parseDeviceOptions, E as deviceNameForSlot, O as describeQuery, S as resolve, T as TOUCHPRESS_DEFAULTS, a as sizeOf, b as parseScreen, d as createScrollSearch, f as directionToward, g as sessionName, h as openSession, i as relativeTo, j as TouchpressError, k as normalizeText, l as captureEvidence, m as probe, n as compareScreenshot, o as toPixelBox, p as formatFailure, r as cropScreenshot, t as preflight, u as createDevice, v as renderTitle, w as evaluate, x as renderScreen, y as silentSink } from "../preflight-DzSceNMx.mjs";
+import { A as textMatch, C as renderScreen, D as parseDeviceOptions, E as deviceNameForSlot, O as describeQuery, S as parseScreen, T as TOUCHPRESS_DEFAULTS, a as sizeOf, b as renderTitle, d as createScrollSearch, f as directionToward, g as sessionName, h as openSession, i as relativeTo, j as TouchpressError, k as normalizeText, l as captureEvidence, m as probe, n as compareScreenshot, o as toPixelBox, p as formatFailure, r as cropScreenshot, t as preflight, u as createDevice, v as describeCheck, w as resolve, x as silentSink, y as evaluate } from "../preflight-B7KrbqSO.mjs";
 export { TOUCHPRESS_DEFAULTS, TouchpressError, captureEvidence, compareScreenshot, createDevice, createScrollSearch, cropScreenshot, describeCheck, describeQuery, deviceNameForSlot, directionToward, evaluate, formatFailure, normalizeText, openSession, parseDeviceOptions, parseScreen, preflight, probe, relativeTo, renderScreen, renderTitle, resolve, sessionName, silentSink, sizeOf, textMatch, toPixelBox };

package/dist/index.d.mts

@@ -1,5 +1,40 @@
-import { E as ReadyQuery, _t as Screen, a as ExpectedValue, at as Filter, b as DeviceSession, ct as TextMatch, g as Locator, h as FilterOptions, ht as Rect, i as ErrorInfo, k as TouchpressOptions, n as PreflightReport, o as TouchpressError, ot as Query, p as Device, pt as Platform, r as preflight, st as Role, t as PreflightDevice, vt as ScreenNode } from "./preflight-CCKYplYi.mjs";
+import { E as ReadyQuery, _t as Screen, a as ExpectedValue, at as Filter, b as DeviceSession, ct as TextMatch, g as Locator, h as FilterOptions, ht as Rect, i as ErrorInfo, k as TouchpressOptions$1, n as PreflightReport, o as TouchpressError, ot as Query, p as Device$1, pt as Platform, r as preflight, st as Role, t as PreflightDevice, vt as ScreenNode } from "./preflight-Czhj9QFM.mjs";
 import { ExpectMatcherState } from "@playwright/test";
+import { FlexibleSchema, LanguageModel } from "ai";
+//#region src/ai/device.d.ts
+/** A loop can run for minutes, so its budget is its own rather than the action timeout a deterministic step takes. */
+type ActOptions = {
+  timeout?: number;
+  maxSteps?: number;
+};
+type ExtractOptions = {
+  timeout?: number;
+};
+type AiDevice = {
+  /**
+   * Drives the app with a model until the instruction is satisfied. Resolves
+   * with the model's summary of what it did, and throws when the model reports
+   * it could not proceed or runs out of steps.
+   */
+  act(instruction: string, options?: ActOptions): Promise<string>;
+  /** Asks a model one question about the current screen and validates the answer against `schema`. */
+  extract<T>(question: string, schema: FlexibleSchema<T>, options?: ExtractOptions): Promise<T>;
+};
+//#endregion
+//#region src/ai/options.d.ts
+/**
+ * The one key `act` and `extract` add to Playwright's `use`. It is not part of
+ * `core/config.ts`, because nothing under `core/` may name an AI SDK type.
+ *
+ * A gateway model id such as `'anthropic/claude-sonnet-5'` and a provider
+ * model instance are both `LanguageModel`, so a config picks either without a
+ * second key. Unset is the default, and it fails at the first `act` rather than
+ * at worker start, so a project that never calls one needs no model.
+ */
+type AiOptions = {
+  aiModel: LanguageModel | undefined;
+};
+//#endregion
 //#region src/playwright/fixtures.d.ts
 /**
  * Touchpress's options and none of its fixtures, for a setup project that reads the
@@ -10,7 +45,7 @@ import { ExpectMatcherState } from "@playwright/test";
  * type, and `parseDeviceOptions` rejects `undefined` by name, so a config that
  * forgot a key and one that never set it fail the same way.
  */
-declare const setupTest: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions & object, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions & TouchpressOptions>;
+declare const setupTest: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions & object, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions & TouchpressOptions$1 & AiOptions>;
 /**
  * `device` is auto so evidence capture runs for every test in a device project,
  * whether or not the body touched it. Its teardown runs before the session's,
@@ -21,8 +56,8 @@ declare const setupTest: import("@playwright/test").TestType<import("@playwright
  * `page` are lazy and non-auto, and nothing here names them.
  */
 declare const test: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions & object & {
-  device: Device;
-}, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions & TouchpressOptions & {
+  device: Device$1 & AiDevice;
+}, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions & TouchpressOptions$1 & AiOptions & {
   session: DeviceSession;
 }>;
 //#endregion
@@ -64,7 +99,7 @@ declare const expect: import("@playwright/test").Expect<{
   toHaveText(this: ExpectMatcherState, locator: Locator, expected: string | RegExp, options?: TextMatcherOptions): Promise<MatcherResult>;
   toHaveValue(this: ExpectMatcherState, locator: Locator, expected: string | RegExp, options?: MatcherOptions): Promise<MatcherResult>;
   toHaveCount(this: ExpectMatcherState, locator: Locator, expected: number, options?: MatcherOptions): Promise<MatcherResult>;
-  toHaveScreenshot(this: ExpectMatcherState, target: Device | Locator, nameOrOptions?: string | ScreenshotOptions, options?: ScreenshotOptions): Promise<{
+  toHaveScreenshot(this: ExpectMatcherState, target: Device$1 | Locator, nameOrOptions?: string | ScreenshotOptions, options?: ScreenshotOptions): Promise<{
     pass: boolean;
     message: () => string;
     name: string;
@@ -73,4 +108,13 @@ declare const expect: import("@playwright/test").Expect<{
   }>;
 }>;
 //#endregion
-export { type Device, type ErrorInfo, type ExpectedValue, type Filter, type FilterOptions, type Locator, type Platform, type PreflightDevice, type PreflightReport, type Query, type ReadyQuery, type Rect, type Role, type Screen, type ScreenNode, type ScreenshotOptions, type TextMatch, TouchpressError, type TouchpressOptions, expect, preflight, setupTest, test };
+//#region src/index.d.ts
+/**
+ * `Device` and `TouchpressOptions` carry the AI surface here and only here.
+ * `touchpress/core` exports the runner-independent pair without it, because
+ * nothing under `core/` may name an AI SDK type.
+ */
+type TouchpressOptions = TouchpressOptions$1 & AiOptions;
+type Device = Device$1 & AiDevice;
+//#endregion
+export { type ActOptions, type AiDevice, type AiOptions, Device, type ErrorInfo, type ExpectedValue, type ExtractOptions, type Filter, type FilterOptions, type Locator, type Platform, type PreflightDevice, type PreflightReport, type Query, type ReadyQuery, type Rect, type Role, type Screen, type ScreenNode, type ScreenshotOptions, type TextMatch, TouchpressError, TouchpressOptions, expect, preflight, setupTest, test };

package/dist/index.mjs

@@ -1,7 +1,500 @@
-import { A as textMatch, D as parseDeviceOptions, S as resolve, T as TOUCHPRESS_DEFAULTS, _ as sleep, a as sizeOf, c as createClient, h as openSession, i as relativeTo, j as TouchpressError, l as captureEvidence, n as compareScreenshot, o as toPixelBox, r as cropScreenshot, s as createAgentDeviceDriver, t as preflight, u as createDevice, y as silentSink } from "./preflight-DzSceNMx.mjs";
+import { A as textMatch, C as renderScreen, D as parseDeviceOptions, S as parseScreen, T as TOUCHPRESS_DEFAULTS, _ as sleep, a as sizeOf, b as renderTitle, c as createClient, h as openSession, i as relativeTo, j as TouchpressError, l as captureEvidence, n as compareScreenshot, o as toPixelBox, r as cropScreenshot, s as createAgentDeviceDriver, t as preflight, u as createDevice, w as resolve, x as silentSink } from "./preflight-B7KrbqSO.mjs";
 import { expect as expect$1, test as test$1 } from "@playwright/test";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname } from "node:path";
+//#region src/ai/tools.ts
+/**
+* The AI SDK is an optional peer, so it is imported here and never at module
+* scope. Importing `touchpress` must not require it, because a project that
+* calls neither `act` nor `extract` never installs it.
+*/
+async function loadAi() {
+	try {
+		return await import("ai");
+	} catch {
+		throw new TouchpressError({ kind: "ai-missing-peer" });
+	}
+}
+const READ = {
+	detail: null,
+	echoesText: false,
+	output: "raw"
+};
+const DEVICE_TOOLS = /* @__PURE__ */ new Map([
+	["snapshot", {
+		detail: null,
+		echoesText: false,
+		output: "screen"
+	}],
+	["press", {
+		detail: "target",
+		echoesText: false,
+		output: "outcome"
+	}],
+	["fill", {
+		detail: "target",
+		echoesText: true,
+		output: "outcome"
+	}],
+	["type", {
+		detail: null,
+		echoesText: true,
+		output: "outcome"
+	}],
+	["scroll", {
+		detail: "direction",
+		echoesText: false,
+		output: "outcome"
+	}],
+	["back", {
+		detail: null,
+		echoesText: false,
+		output: "outcome"
+	}],
+	["wait", READ],
+	["get", READ],
+	["is", READ],
+	["alert", READ]
+]);
+/**
+* Everything an input schema says about which device, daemon, or workspace a
+* command reaches. The session already pins all of it, and a model that could
+* name a daemon could leave the device this test opened.
+*
+* `recordAs` rides along for a different reason: a `fill` carrying it fails
+* unless script recording is armed, which under touchpress it never is.
+*/
+const CUT_KEYS = /* @__PURE__ */ new Set([
+	"udid",
+	"serial",
+	"device",
+	"deviceTarget",
+	"daemonBaseUrl",
+	"daemonAuthToken",
+	"tenant",
+	"runId",
+	"leaseId",
+	"cwd",
+	"debug",
+	"iosSimulatorDeviceSet",
+	"iosXctestrunFile",
+	"iosXctestDerivedDataPath",
+	"iosXctestEnvDir",
+	"androidDeviceAllowlist",
+	"noRecord",
+	"record",
+	"saveScript",
+	"stateDir",
+	"recordAs"
+]);
+/**
+* The tools a model drives the app with, built from agent-device's own command
+* registry so the descriptions and the executors stay upstream's. Three things
+* change: the set is narrowed to the ten commands that perceive and act, every
+* key that could point a command at another device is cut from the input
+* schema, and each executor is wrapped so what crosses to the model is the
+* shape the model can act on. Building them contacts no device.
+*/
+async function createDeviceTools(session, platform) {
+	const { createAgentDeviceTools } = await import("agent-device/ai-sdk");
+	const { jsonSchema, tool } = await loadAi();
+	const { tools } = await createAgentDeviceTools({
+		session,
+		platform
+	});
+	const kept = {};
+	for (const name of DEVICE_TOOLS.keys()) {
+		const built = tools[name];
+		if (built === void 0) throw new Error(`agent-device no longer exposes the "${name}" tool`);
+		kept[name] = tool({
+			description: built.description,
+			inputSchema: jsonSchema(prune(built.inputSchema.jsonSchema)),
+			execute: wrapDeviceTool(name, platform, built.execute)
+		});
+	}
+	return kept;
+}
+/**
+* The one place a command's input and output are reshaped for the model, so the
+* table above stays the only thing that says which command gets which shape.
+*/
+function wrapDeviceTool(name, platform, execute) {
+	const shape = DEVICE_TOOLS.get(name)?.output ?? "raw";
+	return async (input, options) => {
+		const output = await execute(shape === "screen" ? {
+			...asObject(input),
+			forceFull: true
+		} : withRefSigil(input), options);
+		return shape === "screen" ? compactSnapshot(asSnapshot(output), platform) : compactResult(name, output);
+	};
+}
+/**
+* The tree the model reads, in the same listing a failure message prints, one
+* node per line indented by depth. The JSON upstream returns says the same thing
+* with rects, indexes, and flags the model never uses, at three to seven times
+* the size across this repo's fixtures, and its refs arrive without the `@` the
+* action schemas demand.
+*/
+function compactSnapshot(raw, platform) {
+	const screen = parseScreen(raw, platform);
+	const listing = renderScreen(screen);
+	return screen.truncated ? `${listing}\n  ... the tree is truncated, so some nodes are missing` : listing;
+}
+/**
+* What the model needs off an action it just ran: whether it landed, on what,
+* and whether the screen went quiet. Upstream also returns the settle diff, the
+* evidence paths, the resolution, and the cost, which are most of the step's
+* tokens and none of its meaning.
+*/
+function compactResult(name, output) {
+	if (DEVICE_TOOLS.get(name)?.output !== "outcome") return output;
+	if (typeof output !== "object" || output === null) return output;
+	const settle = asObject(output)["settle"];
+	return {
+		...pick(output, "message"),
+		...pick(output, "targetKind"),
+		...typeof settle === "object" && settle !== null ? { settle: {
+			...pick(settle, "settled"),
+			...pick(settle, "waitedMs")
+		} } : {}
+	};
+}
+/**
+* Upstream's own snapshot nodes carry a bare `e4` while the action schemas
+* demand `@e4`, and a model that sends the bare form gets an error it tends to
+* read as "refs do not work here" before falling back to coordinates for the
+* rest of the run. The listing above now prints the `@`, and this catches the
+* model that typed it from memory anyway.
+*/
+function withRefSigil(input) {
+	const target = asObject(asObject(input)["target"]);
+	if (target["kind"] !== "ref") return input;
+	const ref = target["ref"];
+	if (typeof ref !== "string" || ref.startsWith("@")) return input;
+	return {
+		...asObject(input),
+		target: {
+			...target,
+			ref: `@${ref}`
+		}
+	};
+}
+function asObject(input) {
+	return typeof input === "object" && input !== null ? input : {};
+}
+function asSnapshot(output) {
+	if (Array.isArray(asObject(output)["nodes"])) return output;
+	throw new Error("agent-device returned a snapshot without nodes");
+}
+function pick(source, key) {
+	const value = asObject(source)[key];
+	return value === void 0 ? {} : { [key]: value };
+}
+function prune(schema) {
+	if (schema.properties === void 0) return schema;
+	const properties = {};
+	for (const [key, value] of Object.entries(schema.properties)) if (!CUT_KEYS.has(key) && !(key === "target" && isDeviceAlias(value))) properties[key] = value;
+	return {
+		...schema,
+		properties,
+		...schema.required === void 0 ? {} : { required: schema.required.filter((key) => key in properties) }
+	};
+}
+/**
+* Upstream spends `target` twice. On `press`, `fill`, and `get` it is the UI
+* element, a `oneOf` over ref, selector, and point. On the rest it is an alias
+* for `deviceTarget`, an enum of device forms, and a model reading both in one
+* tool set answers the second where the first was meant. The session pins the
+* device, so the alias goes and only the UI target survives.
+*/
+function isDeviceAlias(schema) {
+	return Array.isArray(asObject(schema)["enum"]);
+}
+/** A tool the table does not name still reports, by its name alone. */
+function toolRecord(name, input) {
+	const detail = DEVICE_TOOLS.get(name)?.detail ?? null;
+	const target = detail === null ? null : readText(input, detail);
+	return target === null ? {
+		kind: "tool",
+		name
+	} : {
+		kind: "tool",
+		name,
+		target
+	};
+}
+/**
+* The text a `fill` or a `type` wrote, for the nested step under it. Never
+* masked: the model composed this string and already holds it, so hiding it
+* from the report would cost evidence and buy nothing. A credential belongs in
+* a deterministic `fill(text, { secret: true })` outside `act`.
+*/
+function typedText(name, input) {
+	return DEVICE_TOOLS.get(name)?.echoesText === true ? readText(input, "text") : null;
+}
+function readText(input, key) {
+	const value = asObject(input)[key];
+	return typeof value === "string" && value.length > 0 ? value : null;
+}
+//#endregion
+//#region src/ai/act.ts
+const TRANSCRIPT_RESULT_LIMIT = 2048;
+function instructionsFor(platform) {
+	return [
+		`You are driving a ${platform} mobile app that is already launched and in the foreground.`,
+		"Start with snapshot.",
+		"A snapshot prints one node per line, indented by depth, as in: @e4 [button] \"Sign in\" #signIn",
+		"Use press to tap a node and fill to replace a field's text.",
+		"Both take the ref as { \"kind\": \"ref\", \"ref\": \"@e4\" }, copied exactly as the line printed it.",
+		"Every ref stops working when the next command runs, so snapshot again after each action before you use one.",
+		"Use coordinates only when no line on the snapshot is the thing you need.",
+		"Once a snapshot shows the instruction is satisfied, call done with outcome \"completed\" and a one-line summary. Do not take a second snapshot to double-check.",
+		"If you cannot proceed, call done with outcome \"blocked\" and say what stopped you."
+	].join("\n");
+}
+const EXTRACT_INSTRUCTIONS = [
+	"You are reading one accessibility tree captured from a mobile app.",
+	"Each line is one node, indented by depth, carrying its ref, role, name, and test id, as in: @e4 [button] \"Sign in\" #signIn",
+	"The tree never prints a field's value.",
+	"Answer from what the tree shows, not from what the app is expected to show."
+].join("\n");
+/**
+* Runs the model against the tools until it calls `done`. Nothing here knows
+* about a session, so a test drives it with fake tools and a mock model.
+*
+* A tool error thrown inside `execute` is not caught: the AI SDK hands it back
+* to the model as a tool result, which is what lets it recover from a ref that
+* went stale by taking a fresh snapshot.
+*/
+function runAct(run) {
+	return run.sink.step(renderTitle({
+		kind: "act",
+		instruction: run.instruction
+	}), async () => {
+		const { ToolLoopAgent, hasToolCall, jsonSchema, stepCountIs, tool } = await loadAi();
+		const result = await new ToolLoopAgent({
+			model: run.model,
+			instructions: instructionsFor(run.platform),
+			tools: {
+				...reporting(run.tools, run.sink),
+				done: tool({
+					description: "Finish the instruction. Call this when it is satisfied, or when you cannot proceed.",
+					inputSchema: jsonSchema({
+						type: "object",
+						properties: {
+							outcome: {
+								type: "string",
+								enum: ["completed", "blocked"],
+								description: "completed when the instruction is satisfied, blocked when it is not"
+							},
+							summary: {
+								type: "string",
+								description: "One line saying what you did, or what stopped you."
+							}
+						},
+						required: ["outcome", "summary"],
+						additionalProperties: false
+					})
+				})
+			},
+			stopWhen: [hasToolCall("done"), stepCountIs(run.maxSteps)]
+		}).generate({
+			prompt: run.instruction,
+			abortSignal: AbortSignal.timeout(run.timeout)
+		}).catch(async (error) => {
+			if (!timedOut(error)) throw error;
+			throw new TouchpressError({
+				kind: "ai-timeout",
+				instruction: run.instruction,
+				timeoutMs: run.timeout,
+				screen: await run.screen()
+			});
+		});
+		await run.sink.attach({
+			name: `ai-act-${String(run.attempt)}.json`,
+			contentType: "application/json",
+			body: JSON.stringify({
+				instruction: run.instruction,
+				usage: result.usage,
+				steps: result.steps.map((step) => {
+					const errors = toolErrors(step.content);
+					return {
+						text: step.text,
+						toolCalls: step.toolCalls.map((call) => ({
+							name: call.toolName,
+							input: call.input
+						})),
+						toolResults: step.toolResults.map((toolResult) => ({
+							name: toolResult.toolName,
+							output: clip(toolResult.output)
+						})),
+						...errors.length === 0 ? {} : { toolErrors: errors }
+					};
+				})
+			}, null, 2)
+		});
+		const outcome = outcomeOf(result.steps.at(-1)?.toolCalls);
+		if (outcome === null) throw new TouchpressError({
+			kind: "ai-incomplete",
+			instruction: run.instruction,
+			steps: result.steps.length,
+			screen: await run.screen()
+		});
+		if (outcome.kind === "blocked") throw new TouchpressError({
+			kind: "ai-blocked",
+			instruction: run.instruction,
+			summary: outcome.summary,
+			screen: await run.screen()
+		});
+		return outcome.summary;
+	});
+}
+/** One capture, one question, one answer. No tools, so the model cannot change the screen it is describing. */
+function runExtract(run) {
+	return run.sink.step(renderTitle({
+		kind: "extract",
+		question: run.question
+	}), async () => {
+		const { ToolLoopAgent, Output } = await loadAi();
+		return (await new ToolLoopAgent({
+			model: run.model,
+			instructions: EXTRACT_INSTRUCTIONS,
+			output: Output.object({ schema: run.schema })
+		}).generate({
+			prompt: [
+				`Question: ${run.question}`,
+				``,
+				`Screen:`,
+				run.screen
+			].join("\n"),
+			abortSignal: AbortSignal.timeout(run.timeout)
+		})).output;
+	});
+}
+/** Every model action becomes a step wrapping its own execution, so a report shows the loop as it ran. */
+function reporting(tools, sink) {
+	const wrapped = {};
+	for (const [name, built] of Object.entries(tools)) {
+		const { execute } = built;
+		if (execute === void 0) {
+			wrapped[name] = built;
+			continue;
+		}
+		wrapped[name] = {
+			...built,
+			execute: (input, options) => sink.step(renderTitle(toolRecord(name, input)), async () => {
+				const text = typedText(name, input);
+				if (text !== null) await sink.step(renderTitle({
+					kind: "typed",
+					typed: {
+						kind: "text",
+						value: text
+					}
+				}), () => Promise.resolve(), { box: true });
+				return execute(input, options);
+			})
+		};
+	}
+	return wrapped;
+}
+/**
+* The `done` call is validated here rather than trusted, because a JSON schema
+* handed to `jsonSchema()` describes the tool to the model and validates
+* nothing. A malformed call is a run that never reached an outcome.
+*/
+function outcomeOf(calls) {
+	const call = calls?.find((one) => one.toolName === "done");
+	if (call === void 0 || typeof call.input !== "object" || call.input === null) return null;
+	const summary = Reflect.get(call.input, "summary");
+	if (typeof summary !== "string") return null;
+	const outcome = Reflect.get(call.input, "outcome");
+	if (outcome === "completed") return {
+		kind: "completed",
+		summary
+	};
+	if (outcome === "blocked") return {
+		kind: "blocked",
+		summary
+	};
+	return null;
+}
+/** `AbortSignal.timeout` rejects with a DOMException named TimeoutError, which a provider surfaces as is or as an AbortError. */
+function timedOut(error) {
+	return error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError");
+}
+/**
+* The calls that failed. `step.toolResults` holds only the ones that returned,
+* so a transcript built from it alone shows a loop doing nothing and never says
+* why, which is what a nine-error run looked like from the attachment.
+*/
+function toolErrors(content) {
+	return content.filter((part) => part.type === "tool-error").map((part) => ({
+		name: part.toolName,
+		input: part.input,
+		error: messageOf(part.error)
+	}));
+}
+function messageOf(error) {
+	if (error instanceof Error) return error.message;
+	return typeof error === "string" ? error : JSON.stringify(error) ?? "unknown error";
+}
+function clip(output) {
+	const text = JSON.stringify(output) ?? "undefined";
+	return text.length <= TRANSCRIPT_RESULT_LIMIT ? text : `${text.slice(0, TRANSCRIPT_RESULT_LIMIT)}...`;
+}
+//#endregion
+//#region src/ai/device.ts
+const DEFAULT_ACT_TIMEOUT_MS = 12e4;
+const DEFAULT_ACT_STEPS = 25;
+const DEFAULT_EXTRACT_TIMEOUT_MS = 6e4;
+/**
+* Adds `act` and `extract` to a device without touching what is already there.
+*
+* Both run inside `session.run`, so the whole loop holds the session queue. A
+* test body is sequential anyway, and the model's snapshot refs carry the same
+* rule every deterministic action does: they are valid for the next command
+* only, so nothing else may reach the device in between.
+*/
+function withAi(device, session, sink, model) {
+	let acts = 0;
+	let built = null;
+	const tools = () => built ??= createDeviceTools(session.name, session.options.platform);
+	return {
+		...device,
+		act: async (instruction, options) => {
+			const configured = configuredModel(model);
+			const deviceTools = await tools();
+			return session.run((one) => runAct({
+				model: configured,
+				tools: deviceTools,
+				sink,
+				instruction,
+				platform: session.options.platform,
+				maxSteps: options?.maxSteps ?? DEFAULT_ACT_STEPS,
+				timeout: options?.timeout ?? DEFAULT_ACT_TIMEOUT_MS,
+				screen: async () => renderScreen(await one.capture()),
+				attempt: acts += 1
+			}));
+		},
+		extract: async (question, schema, options) => {
+			const configured = configuredModel(model);
+			return session.run(async (one) => runExtract({
+				model: configured,
+				screen: renderScreen(await one.capture()),
+				question,
+				schema,
+				sink,
+				timeout: options?.timeout ?? DEFAULT_EXTRACT_TIMEOUT_MS
+			}));
+		}
+	};
+}
+/** Checked before the queue, so a project that forgot the key fails at once rather than holding the session. */
+function configuredModel(model) {
+	if (model === void 0) throw new TouchpressError({ kind: "ai-not-configured" });
+	return model;
+}
+//#endregion
 //#region src/playwright/fixtures.ts
 const SESSION_FIXTURE_TIMEOUT_MS = 18e4;
 const DEVICE_FIXTURE_TIMEOUT_MS = 12e4;
@@ -62,6 +555,10 @@ const setupTest = test$1.extend({
 	sessionPrefix: [TOUCHPRESS_DEFAULTS.sessionPrefix, {
 		option: true,
 		scope: "worker"
+	}],
+	aiModel: [void 0, {
+		option: true,
+		scope: "worker"
 	}]
 });
 /** The worker session already opened the app with a relaunch, so the first test skips one. */
@@ -105,11 +602,11 @@ const test = setupTest.extend({
 		scope: "worker",
 		timeout: SESSION_FIXTURE_TIMEOUT_MS
 	}],
-	device: [async ({ session }, use, testInfo) => {
+	device: [async ({ session, aiModel }, use, testInfo) => {
 		const sink = playwrightSink();
 		if (session.options.relaunch === "per-test" && startedTests.has(session)) await session.relaunch(sink);
 		startedTests.add(session);
-		await use(createDevice(session, sink));
+		await use(withAi(createDevice(session, sink), session, sink, aiModel));
 		if (shouldCapture(testInfo, session.options.evidence)) await captureEvidence(session, sink);
 	}, {
 		auto: true,

package/dist/{preflight-DzSceNMx.mjs → preflight-B7KrbqSO.mjs}

@@ -59,6 +59,36 @@ function formatError(info) {
 			info.screen
 		].join("\n");
 		case "driver": return `${info.command} failed: ${describeFailure(info.failure)}`;
+		case "ai-not-configured": return ["device.act and device.extract need a model.", "Set use.aiModel to a gateway model id, such as 'anthropic/claude-sonnet-5', or to a provider model instance."].join("\n");
+		case "ai-missing-peer": return ["device.act and device.extract need the optional peer dependency 'ai'.", "Install it with: pnpm add -D ai"].join("\n");
+		case "ai-blocked": return [
+			`act stopped without finishing: ${info.summary}`,
+			``,
+			`Instruction: ${info.instruction}`,
+			``,
+			`Screen:`,
+			info.screen
+		].join("\n");
+		case "ai-incomplete": return [
+			`act ran ${String(info.steps)} steps without reaching an outcome.`,
+			``,
+			`Instruction: ${info.instruction}`,
+			``,
+			"Raise maxSteps, or split the instruction into smaller ones.",
+			``,
+			`Screen:`,
+			info.screen
+		].join("\n");
+		case "ai-timeout": return [
+			`act ran out of its ${String(info.timeoutMs)}ms budget before reaching an outcome.`,
+			``,
+			`Instruction: ${info.instruction}`,
+			``,
+			"Raise the act timeout, and the test timeout with it.",
+			``,
+			`Screen:`,
+			info.screen
+		].join("\n");
 		default: throw new Error(`unhandled error info ${JSON.stringify(info)}`);
 	}
 }
@@ -355,89 +385,6 @@ function fail(field, detail) {
 	});
 }
 //#endregion
-//#region src/core/checks.ts
-/**
-* A `many` outcome never passes anything but `toHaveCount`. An ambiguous locator
-* is a strictness violation, and it reports through the same message path as a
-* plain mismatch rather than guessing which node was meant.
-*/
-function evaluate(check, resolution) {
-	if (check.name === "toHaveCount") {
-		const count = countOf(resolution);
-		return {
-			pass: count === check.expected,
-			actual: String(count)
-		};
-	}
-	if (resolution.outcome === "none") return {
-		pass: false,
-		actual: null
-	};
-	if (resolution.outcome === "many") return {
-		pass: false,
-		actual: `${String(resolution.nodes.length)} matching nodes`
-	};
-	const node = resolution.node;
-	switch (check.name) {
-		case "toBeVisible": return {
-			pass: true,
-			actual: describeNode(node)
-		};
-		case "toHaveText": {
-			const text = node.name ?? node.value;
-			return {
-				pass: matchesText(check.expected, text),
-				actual: text === null ? null : `"${text}"`
-			};
-		}
-		case "toHaveValue": return {
-			pass: matchesText(check.expected, node.value),
-			actual: node.value === null ? null : `"${node.value}"`
-		};
-		case "toBeEnabled": return {
-			pass: node.enabled,
-			actual: node.enabled ? "enabled" : "disabled"
-		};
-		case "toBeSelected": return {
-			pass: node.selected,
-			actual: node.selected ? "selected" : "not selected"
-		};
-		case "toBeFocused": return {
-			pass: node.focused,
-			actual: node.focused ? "focused" : "not focused"
-		};
-		default: throw new Error(`unhandled check ${JSON.stringify(check)}`);
-	}
-}
-function countOf(resolution) {
-	switch (resolution.outcome) {
-		case "one": return 1;
-		case "none": return 0;
-		case "many": return resolution.nodes.length;
-		default: throw new Error(`unhandled resolution ${JSON.stringify(resolution)}`);
-	}
-}
-/** The `Expected:` line. */
-function describeCheck(check) {
-	switch (check.name) {
-		case "toBeVisible": return "visible";
-		case "toHaveText": return `text ${describeExpected(check.expected)}`;
-		case "toHaveValue": return `value ${describeExpected(check.expected)}`;
-		case "toBeEnabled": return "enabled";
-		case "toBeSelected": return "selected";
-		case "toBeFocused": return "focused";
-		case "toHaveCount": return `count ${String(check.expected)}`;
-		default: throw new Error(`unhandled check ${JSON.stringify(check)}`);
-	}
-}
-function describeExpected(match) {
-	return match.kind === "regex" ? String(match.value) : `"${match.value}"`;
-}
-function describeNode(node) {
-	const name = node.name === null ? "" : ` "${node.name}"`;
-	return `${node.ref} [${node.role}]${name}`;
-}
-//#endregion
 //#region src/core/screen.ts
 const IOS_ROLES = {
 	Application: "application",
@@ -716,6 +663,7 @@ function renderNode(node) {
 //#endregion
 //#region src/core/report.ts
 const FILL_TEXT_LIMIT = 40;
+const PROMPT_LIMIT = 80;
 /** Rendered here rather than in an adapter so every runner produces the same text. */
 function renderTitle(record) {
 	switch (record.kind) {
@@ -729,18 +677,21 @@ function renderTitle(record) {
 		case "relaunch": return `relaunch ${record.app}`;
 		case "dismiss-overlay": return "dismiss the React Native dev overlay";
 		case "screenshot": return `screenshot ${record.path}`;
+		case "act": return `act "${truncate(record.instruction, PROMPT_LIMIT)}"`;
+		case "extract": return `extract "${truncate(record.question, PROMPT_LIMIT)}"`;
+		case "tool": return record.target === void 0 ? record.name : `${record.name} ${record.target}`;
 		default: throw new Error(`unhandled action record ${JSON.stringify(record)}`);
 	}
 }
 function renderTyped(typed) {
 	switch (typed.kind) {
-		case "text": return `type "${truncate(typed.value)}"`;
+		case "text": return `type "${truncate(typed.value, FILL_TEXT_LIMIT)}"`;
 		case "hidden": return `type ${String(typed.length)} characters`;
 		default: throw new Error(`unhandled typed value ${JSON.stringify(typed)}`);
 	}
 }
-function truncate(text) {
-	return text.length <= FILL_TEXT_LIMIT ? text : `${text.slice(0, FILL_TEXT_LIMIT)}...`;
+function truncate(text, limit) {
+	return text.length <= limit ? text : `${text.slice(0, limit)}...`;
 }
 /** Discards everything. The default for scripts, unit tests, and runners with no reporting. */
 const silentSink = {
@@ -750,6 +701,89 @@ const silentSink = {
 	outputPath: (fileName) => fileName
 };
 //#endregion
+//#region src/core/checks.ts
+/**
+* A `many` outcome never passes anything but `toHaveCount`. An ambiguous locator
+* is a strictness violation, and it reports through the same message path as a
+* plain mismatch rather than guessing which node was meant.
+*/
+function evaluate(check, resolution) {
+	if (check.name === "toHaveCount") {
+		const count = countOf(resolution);
+		return {
+			pass: count === check.expected,
+			actual: String(count)
+		};
+	}
+	if (resolution.outcome === "none") return {
+		pass: false,
+		actual: null
+	};
+	if (resolution.outcome === "many") return {
+		pass: false,
+		actual: `${String(resolution.nodes.length)} matching nodes`
+	};
+	const node = resolution.node;
+	switch (check.name) {
+		case "toBeVisible": return {
+			pass: true,
+			actual: describeNode(node)
+		};
+		case "toHaveText": {
+			const text = node.name ?? node.value;
+			return {
+				pass: matchesText(check.expected, text),
+				actual: text === null ? null : `"${text}"`
+			};
+		}
+		case "toHaveValue": return {
+			pass: matchesText(check.expected, node.value),
+			actual: node.value === null ? null : `"${node.value}"`
+		};
+		case "toBeEnabled": return {
+			pass: node.enabled,
+			actual: node.enabled ? "enabled" : "disabled"
+		};
+		case "toBeSelected": return {
+			pass: node.selected,
+			actual: node.selected ? "selected" : "not selected"
+		};
+		case "toBeFocused": return {
+			pass: node.focused,
+			actual: node.focused ? "focused" : "not focused"
+		};
+		default: throw new Error(`unhandled check ${JSON.stringify(check)}`);
+	}
+}
+function countOf(resolution) {
+	switch (resolution.outcome) {
+		case "one": return 1;
+		case "none": return 0;
+		case "many": return resolution.nodes.length;
+		default: throw new Error(`unhandled resolution ${JSON.stringify(resolution)}`);
+	}
+}
+/** The `Expected:` line. */
+function describeCheck(check) {
+	switch (check.name) {
+		case "toBeVisible": return "visible";
+		case "toHaveText": return `text ${describeExpected(check.expected)}`;
+		case "toHaveValue": return `value ${describeExpected(check.expected)}`;
+		case "toBeEnabled": return "enabled";
+		case "toBeSelected": return "selected";
+		case "toBeFocused": return "focused";
+		case "toHaveCount": return `count ${String(check.expected)}`;
+		default: throw new Error(`unhandled check ${JSON.stringify(check)}`);
+	}
+}
+function describeExpected(match) {
+	return match.kind === "regex" ? String(match.value) : `"${match.value}"`;
+}
+function describeNode(node) {
+	const name = node.name === null ? "" : ` "${node.name}"`;
+	return `${node.ref} [${node.role}]${name}`;
+}
+//#endregion
 //#region src/core/session.ts
 const READY_POLL_MS = 250;
 const SNAPSHOT_TIMEOUT_MS = 15e3;
@@ -1862,4 +1896,4 @@ function notBooted(platform, name, booted) {
 	return `No booted ${platform} device is named '${name}'. Booted right now: ${booted.map((device) => `'${device.name}'`).join(", ")}. Set use.deviceName to one of those or boot '${name}'.`;
 }
 //#endregion
-export { textMatch as A, describeCheck as C, parseDeviceOptions as D, deviceNameForSlot as E, describeQuery as O, resolve as S, TOUCHPRESS_DEFAULTS as T, sleep as _, sizeOf as a, parseScreen as b, createClient as c, createScrollSearch as d, directionToward as f, sessionName as g, openSession as h, relativeTo as i, TouchpressError as j, normalizeText as k, captureEvidence as l, probe as m, compareScreenshot as n, toPixelBox as o, formatFailure as p, cropScreenshot as r, createAgentDeviceDriver as s, preflight as t, createDevice as u, renderTitle as v, evaluate as w, renderScreen as x, silentSink as y };
+export { textMatch as A, renderScreen as C, parseDeviceOptions as D, deviceNameForSlot as E, describeQuery as O, parseScreen as S, TOUCHPRESS_DEFAULTS as T, sleep as _, sizeOf as a, renderTitle as b, createClient as c, createScrollSearch as d, directionToward as f, sessionName as g, openSession as h, relativeTo as i, TouchpressError as j, normalizeText as k, captureEvidence as l, probe as m, compareScreenshot as n, toPixelBox as o, formatFailure as p, cropScreenshot as r, createAgentDeviceDriver as s, preflight as t, createDevice as u, describeCheck as v, resolve as w, silentSink as x, evaluate as y };

package/dist/{preflight-CCKYplYi.d.mts → preflight-Czhj9QFM.d.mts}

@@ -448,6 +448,18 @@ type ActionRecord = {
 } | {
   readonly kind: 'screenshot';
   readonly path: string;
+} | {
+  readonly kind: 'act';
+  readonly instruction: string;
+} | {
+  readonly kind: 'extract';
+  readonly question: string;
+} |
+/** One command the model ran inside an `act`. `target` is its snapshot ref, or a scroll's direction. */
+{
+  readonly kind: 'tool';
+  readonly name: string;
+  readonly target?: string;
 };
 /** A boxed step reports as one line rather than as something to open. */
 type StepOptions = {
@@ -825,6 +837,26 @@ type ErrorInfo = {
   readonly kind: 'driver';
   readonly command: string;
   readonly failure: DeviceFailure;
+} | {
+  readonly kind: 'ai-not-configured';
+} | {
+  readonly kind: 'ai-missing-peer';
+} | {
+  readonly kind: 'ai-blocked';
+  readonly instruction: string;
+  /** The model's own account of what stopped it. */
+  readonly summary: string;
+  readonly screen: string;
+} | {
+  readonly kind: 'ai-incomplete';
+  readonly instruction: string;
+  readonly steps: number;
+  readonly screen: string;
+} | {
+  readonly kind: 'ai-timeout';
+  readonly instruction: string;
+  readonly timeoutMs: number;
+  readonly screen: string;
 };
 declare class TouchpressError extends Error {
   readonly info: ErrorInfo;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "touchpress",
-  "version": "0.0.2",
+  "version": "0.1.0",
   "description": "End-to-end testing for mobile apps.",
   "homepage": "https://github.com/wobsoriano/touchpress#readme",
   "bugs": {
@@ -33,13 +33,21 @@
     "@playwright/test": "1.63.0",
     "@types/node": "^26.1.1",
     "@types/pngjs": "^6.0.5",
+    "ai": "^7.0.93",
     "bumpp": "^11.1.0",
     "typescript": "^7.0.2",
     "vite": "npm:@voidzero-dev/vite-plus-core@0.3.0",
-    "vite-plus": "0.3.0"
+    "vite-plus": "0.3.0",
+    "zod": "^4.5.4"
   },
   "peerDependencies": {
-    "@playwright/test": ">=1.63"
+    "@playwright/test": ">=1.63",
+    "ai": "^6.0.0 || ^7.0.0"
+  },
+  "peerDependenciesMeta": {
+    "ai": {
+      "optional": true
+    }
   },
   "devEngines": {
     "packageManager": {