touchpress 0.0.2 → 0.1.1

package/README.md

@@ -66,6 +66,19 @@ npx playwright test --project=ios
 
 `preflight` reads a project's options and reports whether the device they name is booted, so a missing simulator fails once, in about a second, instead of once per test after the launch timeout. Wire it as a setup project per platform that the device projects depend on. [Basics](https://github.com/wobsoriano/touchpress/blob/main/docs/basics.md) has the spec, and [`apps/e2e/e2e/preflight.setup.mts`](https://github.com/wobsoriano/touchpress/blob/main/apps/e2e/e2e/preflight.setup.mts) is a working one.
 
+### Drive a step with a model
+
+`device.act` takes an instruction in English and drives the app until it is satisfied. `device.extract` asks one question about the screen and returns a typed answer.
+
+```ts
+test('sign in', async ({ device }) => {
+  await device.act('Sign in with the email rob@example.com and the password hunter2');
+  await expect(device.getByTestId('greeting')).toHaveText('Hi, Rob');
+});
+```
+
+Set `use.aiModel` to a gateway model id or a provider model instance, and install `ai`, an optional peer dependency. A loop can run for minutes, so raise the test timeout on any spec that calls `act`. The example uses the sample app's fake account. Keep real credentials in deterministic `fill(text, { secret: true })` calls outside `act`, and assert deterministically afterwards. [AI](https://github.com/wobsoriano/touchpress/blob/main/docs/ai.md) covers `act`, `extract`, the tools the model gets, and the report.
+
 ## Run the sample project
 
 `apps/e2e` is an Expo app with a home, login, and profile route and a fake sign-in. It is the app touchpress is tested against. Build the library first with `vp run -r build` so the app can resolve `dist`, then run these from `apps/e2e`:
@@ -93,6 +106,7 @@ Leave Metro running for the whole suite. The first build takes several minutes.
 - [Configuration](https://github.com/wobsoriano/touchpress/blob/main/docs/configuration.md)
 - [Locators](https://github.com/wobsoriano/touchpress/blob/main/docs/locators.md)
 - [Assertions](https://github.com/wobsoriano/touchpress/blob/main/docs/assertions.md)
+- [AI](https://github.com/wobsoriano/touchpress/blob/main/docs/ai.md)
 - [Lifecycle](https://github.com/wobsoriano/touchpress/blob/main/docs/lifecycle.md)
 - [Continuous integration](https://github.com/wobsoriano/touchpress/blob/main/docs/ci.md)
 

package/dist/core/index.d.mts

@@ -1,4 +1,4 @@
-import { $ as Tree, A as deviceNameForSlot, B as ProbeResult, C as openSession, D as ResolvedOptions, E as ReadyQuery, F as StepOptions, G as CaptureOptions, H as formatFailure, I as Typed, J as DeviceInfo, K as DeviceDriver, L as renderTitle, M as ActionRecord, N as ActionSink, O as TOUCHPRESS_DEFAULTS, P as EvidenceFile, Q as Settled, R as silentSink, S as SessionState, T as DeviceChoice, U as probe, V as ProbeTarget, W as Binding, X as OpenRequest, Y as DeviceSelection, Z as ScrollDirection, _ as RoleOptions, _t as Screen, a as ExpectedValue, at as Filter, b as DeviceSession, bt as renderScreen, c as ScrollSearch, ct as TextMatch, d as directionToward, dt as textMatch, et as Check, f as ActionOptions, ft as PinnedRef, g as Locator, gt as Resolution, h as FilterOptions, ht as Rect, i as ErrorInfo, it as evaluate, j as parseDeviceOptions, k as TouchpressOptions, l as ScrollTrail, lt as describeQuery, m as FillOptions, mt as RawSnapshot, n as PreflightReport, nt as Verdict, o as TouchpressError, ot as Query, p as Device, pt as Platform, q as DeviceFailure, r as preflight, rt as describeCheck, s as ScrollDevice, st as Role, t as PreflightDevice, tt as CheckName, u as createScrollSearch, ut as normalizeText, v as TextOptions, vt as ScreenNode, w as sessionName, x as OpenSessionInput, xt as resolve, y as createDevice, yt as parseScreen, z as ProbeOptions } from "../preflight-CCKYplYi.mjs";
+import { $ as Tree, A as deviceNameForSlot, B as ProbeResult, C as openSession, D as ResolvedOptions, E as ReadyQuery, F as StepOptions, G as CaptureOptions, H as formatFailure, I as Typed, J as DeviceInfo, K as DeviceDriver, L as renderTitle, M as ActionRecord, N as ActionSink, O as TOUCHPRESS_DEFAULTS, P as EvidenceFile, Q as Settled, R as silentSink, S as SessionState, T as DeviceChoice, U as probe, V as ProbeTarget, W as Binding, X as OpenRequest, Y as DeviceSelection, Z as ScrollDirection, _ as RoleOptions, _t as Screen, a as ExpectedValue, at as Filter, b as DeviceSession, bt as renderScreen, c as ScrollSearch, ct as TextMatch, d as directionToward, dt as textMatch, et as Check, f as ActionOptions, ft as PinnedRef, g as Locator, gt as Resolution, h as FilterOptions, ht as Rect, i as ErrorInfo, it as evaluate, j as parseDeviceOptions, k as TouchpressOptions, l as ScrollTrail, lt as describeQuery, m as FillOptions, mt as RawSnapshot, n as PreflightReport, nt as Verdict, o as TouchpressError, ot as Query, p as Device, pt as Platform, q as DeviceFailure, r as preflight, rt as describeCheck, s as ScrollDevice, st as Role, t as PreflightDevice, tt as CheckName, u as createScrollSearch, ut as normalizeText, v as TextOptions, vt as ScreenNode, w as sessionName, x as OpenSessionInput, xt as resolve, y as createDevice, yt as parseScreen, z as ProbeOptions } from "../preflight-Czhj9QFM.mjs";
 //#region src/core/evidence.d.ts
 /**
  * Never throws. A capture that fails records a note and returns, because masking
@@ -52,9 +52,11 @@ type Comparison = {
  */
 declare function compareScreenshot(expected: Buffer, actual: Buffer, options: CompareOptions): Comparison;
 /**
- * The box is clamped to the image, because a rect comes from a snapshot and a
+ * The box is clipped to the image, because a rect comes from a snapshot and a
  * screenshot is a separate capture. A control flush against the bottom edge can
- * round a pixel past it, and that is not a reason to fail an assertion.
+ * round a pixel past it, and that is not a reason to fail an assertion. A box
+ * with no pixel inside the image is a different thing, a rect the screenshot
+ * does not show, and cutting a placeholder out of the corner would hide that.
  */
 declare function cropScreenshot(source: Buffer, box: PixelBox): Buffer;
 declare function sizeOf(source: Buffer): Size;

package/dist/core/index.mjs

@@ -1,2 +1,2 @@
-import { A as textMatch, C as describeCheck, D as parseDeviceOptions, E as deviceNameForSlot, O as describeQuery, S as resolve, T as TOUCHPRESS_DEFAULTS, a as sizeOf, b as parseScreen, d as createScrollSearch, f as directionToward, g as sessionName, h as openSession, i as relativeTo, j as TouchpressError, k as normalizeText, l as captureEvidence, m as probe, n as compareScreenshot, o as toPixelBox, p as formatFailure, r as cropScreenshot, t as preflight, u as createDevice, v as renderTitle, w as evaluate, x as renderScreen, y as silentSink } from "../preflight-DzSceNMx.mjs";
+import { A as normalizeText, C as parseScreen, D as deviceNameForSlot, E as TOUCHPRESS_DEFAULTS, M as TouchpressError, O as parseDeviceOptions, S as silentSink, T as resolve, a as sizeOf, d as createScrollSearch, f as directionToward, g as evaluate, h as describeCheck, i as relativeTo, j as textMatch, k as describeQuery, l as captureEvidence, m as probe, n as compareScreenshot, o as toPixelBox, p as formatFailure, r as cropScreenshot, t as preflight, u as createDevice, v as openSession, w as renderScreen, x as renderTitle, y as sessionName } from "../preflight-aUba9u1k.mjs";
 export { TOUCHPRESS_DEFAULTS, TouchpressError, captureEvidence, compareScreenshot, createDevice, createScrollSearch, cropScreenshot, describeCheck, describeQuery, deviceNameForSlot, directionToward, evaluate, formatFailure, normalizeText, openSession, parseDeviceOptions, parseScreen, preflight, probe, relativeTo, renderScreen, renderTitle, resolve, sessionName, silentSink, sizeOf, textMatch, toPixelBox };

package/dist/index.d.mts

@@ -1,5 +1,66 @@
-import { E as ReadyQuery, _t as Screen, a as ExpectedValue, at as Filter, b as DeviceSession, ct as TextMatch, g as Locator, h as FilterOptions, ht as Rect, i as ErrorInfo, k as TouchpressOptions, n as PreflightReport, o as TouchpressError, ot as Query, p as Device, pt as Platform, r as preflight, st as Role, t as PreflightDevice, vt as ScreenNode } from "./preflight-CCKYplYi.mjs";
+import { E as ReadyQuery, _t as Screen, a as ExpectedValue, at as Filter, b as DeviceSession, ct as TextMatch, g as Locator, h as FilterOptions, ht as Rect, i as ErrorInfo, k as TouchpressOptions$1, n as PreflightReport, o as TouchpressError, ot as Query, p as Device$1, pt as Platform, r as preflight, st as Role, t as PreflightDevice, vt as ScreenNode } from "./preflight-Czhj9QFM.mjs";
 import { ExpectMatcherState } from "@playwright/test";
+//#region src/ai/options.d.ts
+/**
+ * What `use.aiModel` accepts, spelled structurally so the main entry's
+ * declarations never import from `ai`. A gateway model id such as
+ * `'anthropic/claude-sonnet-5'` and a provider model instance both fit, and
+ * every AI SDK language model, v2 through v4, carries these three fields.
+ */
+type AiModel = string | {
+  readonly specificationVersion: string;
+  readonly provider: string;
+  readonly modelId: string;
+};
+/**
+ * The one key `act` and `extract` add to Playwright's `use`. It is not part of
+ * `core/config.ts`, because nothing under `core/` may name an AI SDK type.
+ *
+ * Unset is the default, and it fails at the first `act` rather than at worker
+ * start, so a project that never calls one needs no model.
+ */
+type AiOptions = {
+  aiModel: AiModel | undefined;
+};
+//#endregion
+//#region src/ai/device.d.ts
+/** A loop can run for minutes, so its budget is its own rather than the action timeout a deterministic step takes. */
+type ActOptions = {
+  timeout?: number;
+  maxSteps?: number;
+};
+type ExtractOptions = {
+  timeout?: number;
+};
+/**
+ * What `extract` accepts, spelled structurally so the main entry's declarations
+ * never import from `ai`. The first branch is a Standard Schema, which Zod
+ * 3.25+, Zod 4, and Valibot implement, with `T` read off its output type. The
+ * second is the AI SDK's own `Schema`, which `jsonSchema()` returns.
+ */
+type ExtractSchema<T> = {
+  readonly '~standard': {
+    readonly version: 1;
+    readonly vendor: string;
+    readonly types?: {
+      readonly output: T;
+    };
+  };
+} | {
+  readonly jsonSchema: unknown;
+  readonly _type?: T;
+};
+type AiDevice = {
+  /**
+   * Drives the app with a model until the instruction is satisfied. Resolves
+   * with the model's summary of what it did, and throws when the model reports
+   * it could not proceed or runs out of steps.
+   */
+  act(instruction: string, options?: ActOptions): Promise<string>;
+  /** Asks a model one question about the current screen and validates the answer against `schema`. */
+  extract<T>(question: string, schema: ExtractSchema<T>, options?: ExtractOptions): Promise<T>;
+};
+//#endregion
 //#region src/playwright/fixtures.d.ts
 /**
  * Touchpress's options and none of its fixtures, for a setup project that reads the
@@ -10,7 +71,7 @@ import { ExpectMatcherState } from "@playwright/test";
  * type, and `parseDeviceOptions` rejects `undefined` by name, so a config that
  * forgot a key and one that never set it fail the same way.
  */
-declare const setupTest: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions & object, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions & TouchpressOptions>;
+declare const setupTest: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions & object, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions & TouchpressOptions$1 & AiOptions>;
 /**
  * `device` is auto so evidence capture runs for every test in a device project,
  * whether or not the body touched it. Its teardown runs before the session's,
@@ -21,8 +82,8 @@ declare const setupTest: import("@playwright/test").TestType<import("@playwright
  * `page` are lazy and non-auto, and nothing here names them.
  */
 declare const test: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions & object & {
-  device: Device;
-}, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions & TouchpressOptions & {
+  device: Device$1 & AiDevice;
+}, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions & TouchpressOptions$1 & AiOptions & {
   session: DeviceSession;
 }>;
 //#endregion
@@ -64,7 +125,7 @@ declare const expect: import("@playwright/test").Expect<{
   toHaveText(this: ExpectMatcherState, locator: Locator, expected: string | RegExp, options?: TextMatcherOptions): Promise<MatcherResult>;
   toHaveValue(this: ExpectMatcherState, locator: Locator, expected: string | RegExp, options?: MatcherOptions): Promise<MatcherResult>;
   toHaveCount(this: ExpectMatcherState, locator: Locator, expected: number, options?: MatcherOptions): Promise<MatcherResult>;
-  toHaveScreenshot(this: ExpectMatcherState, target: Device | Locator, nameOrOptions?: string | ScreenshotOptions, options?: ScreenshotOptions): Promise<{
+  toHaveScreenshot(this: ExpectMatcherState, target: Device$1 | Locator, nameOrOptions?: string | ScreenshotOptions, options?: ScreenshotOptions): Promise<{
     pass: boolean;
     message: () => string;
     name: string;
@@ -73,4 +134,13 @@ declare const expect: import("@playwright/test").Expect<{
   }>;
 }>;
 //#endregion
-export { type Device, type ErrorInfo, type ExpectedValue, type Filter, type FilterOptions, type Locator, type Platform, type PreflightDevice, type PreflightReport, type Query, type ReadyQuery, type Rect, type Role, type Screen, type ScreenNode, type ScreenshotOptions, type TextMatch, TouchpressError, type TouchpressOptions, expect, preflight, setupTest, test };
+//#region src/index.d.ts
+/**
+ * `Device` and `TouchpressOptions` carry the AI surface here and only here.
+ * `touchpress/core` exports the runner-independent pair without it, because
+ * nothing under `core/` may name an AI SDK type.
+ */
+type TouchpressOptions = TouchpressOptions$1 & AiOptions;
+type Device = Device$1 & AiDevice;
+//#endregion
+export { type ActOptions, type AiDevice, type AiModel, type AiOptions, Device, type ErrorInfo, type ExpectedValue, type ExtractOptions, type ExtractSchema, type Filter, type FilterOptions, type Locator, type Platform, type PreflightDevice, type PreflightReport, type Query, type ReadyQuery, type Rect, type Role, type Screen, type ScreenNode, type ScreenshotOptions, type TextMatch, TouchpressError, TouchpressOptions, expect, preflight, setupTest, test };

package/dist/index.mjs

@@ -1,7 +1,524 @@
-import { A as textMatch, D as parseDeviceOptions, S as resolve, T as TOUCHPRESS_DEFAULTS, _ as sleep, a as sizeOf, c as createClient, h as openSession, i as relativeTo, j as TouchpressError, l as captureEvidence, n as compareScreenshot, o as toPixelBox, r as cropScreenshot, s as createAgentDeviceDriver, t as preflight, u as createDevice, y as silentSink } from "./preflight-DzSceNMx.mjs";
+import { C as parseScreen, E as TOUCHPRESS_DEFAULTS, M as TouchpressError, O as parseDeviceOptions, S as silentSink, T as resolve, _ as createQueue, a as sizeOf, b as sleep, c as createClient, i as relativeTo, j as textMatch, l as captureEvidence, n as compareScreenshot, o as toPixelBox, r as cropScreenshot, s as createAgentDeviceDriver, t as preflight, u as createDevice, v as openSession, w as renderScreen, x as renderTitle } from "./preflight-aUba9u1k.mjs";
 import { expect as expect$1, test as test$1 } from "@playwright/test";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname } from "node:path";
+//#region src/ai/tools.ts
+/**
+* The AI SDK is an optional peer, so it is imported here and never at module
+* scope. Importing `touchpress` must not require it, because a project that
+* calls neither `act` nor `extract` never installs it.
+*/
+async function loadAi() {
+	try {
+		return await import("ai");
+	} catch {
+		throw new TouchpressError({ kind: "ai-missing-peer" });
+	}
+}
+const READ = {
+	detail: null,
+	echoesText: false,
+	output: "raw"
+};
+const DEVICE_TOOLS = /* @__PURE__ */ new Map([
+	["snapshot", {
+		detail: null,
+		echoesText: false,
+		output: "screen"
+	}],
+	["press", {
+		detail: "target",
+		echoesText: false,
+		output: "outcome"
+	}],
+	["fill", {
+		detail: "target",
+		echoesText: true,
+		output: "outcome"
+	}],
+	["type", {
+		detail: null,
+		echoesText: true,
+		output: "outcome"
+	}],
+	["scroll", {
+		detail: "direction",
+		echoesText: false,
+		output: "outcome"
+	}],
+	["back", {
+		detail: null,
+		echoesText: false,
+		output: "outcome"
+	}],
+	["wait", READ],
+	["get", READ],
+	["is", READ],
+	["alert", READ]
+]);
+/**
+* Everything an input schema says about which device, daemon, or workspace a
+* command reaches. The session already pins all of it, and a model that could
+* name a daemon could leave the device this test opened.
+*
+* `recordAs` rides along for a different reason: a `fill` carrying it fails
+* unless script recording is armed, which under touchpress it never is.
+*/
+const CUT_KEYS = /* @__PURE__ */ new Set([
+	"udid",
+	"serial",
+	"device",
+	"deviceTarget",
+	"daemonBaseUrl",
+	"daemonAuthToken",
+	"tenant",
+	"runId",
+	"leaseId",
+	"cwd",
+	"debug",
+	"iosSimulatorDeviceSet",
+	"iosXctestrunFile",
+	"iosXctestDerivedDataPath",
+	"iosXctestEnvDir",
+	"androidDeviceAllowlist",
+	"noRecord",
+	"record",
+	"saveScript",
+	"stateDir",
+	"recordAs"
+]);
+/**
+* The tools a model drives the app with, built from agent-device's own command
+* registry so the descriptions and the executors stay upstream's. Three things
+* change: the set is narrowed to the ten commands that perceive and act, every
+* key that could point a command at another device is cut from the input
+* schema, and each executor is wrapped so what crosses to the model is the
+* shape the model can act on. Building them contacts no device.
+*/
+async function createDeviceTools(session, platform) {
+	const { createAgentDeviceTools } = await import("agent-device/ai-sdk");
+	const { jsonSchema, tool } = await loadAi();
+	const { tools } = await createAgentDeviceTools({
+		session,
+		platform
+	});
+	const kept = {};
+	for (const name of DEVICE_TOOLS.keys()) {
+		const built = tools[name];
+		if (built === void 0) throw new Error(`agent-device no longer exposes the "${name}" tool`);
+		const schema = prune(built.inputSchema.jsonSchema);
+		kept[name] = tool({
+			description: built.description,
+			inputSchema: jsonSchema(schema),
+			execute: wrapDeviceTool(name, platform, built.execute, new Set(Object.keys(schema.properties ?? {})))
+		});
+	}
+	return kept;
+}
+/**
+* The one place a command's input and output are reshaped for the model, so the
+* table above stays the only thing that says which command gets which shape.
+*
+* `accepted` is the key set of the pruned schema. A schema handed to
+* `jsonSchema()` describes the tool and validates nothing, so a model that
+* sends a key the schema no longer lists would otherwise have it forwarded.
+*/
+function wrapDeviceTool(name, platform, execute, accepted) {
+	const shape = DEVICE_TOOLS.get(name)?.output ?? "raw";
+	return async (input, options) => {
+		const declared = Object.fromEntries(Object.entries(asObject(input)).filter(([key]) => accepted.has(key)));
+		const output = await execute(shape === "screen" ? {
+			...declared,
+			forceFull: true
+		} : withRefSigil(declared), options);
+		return shape === "screen" ? compactSnapshot(asSnapshot(output), platform) : compactResult(name, output);
+	};
+}
+/**
+* The tree the model reads, in the same listing a failure message prints, one
+* node per line indented by depth. The JSON upstream returns says the same thing
+* with rects, indexes, and flags the model never uses, at three to seven times
+* the size across this repo's fixtures, and its refs arrive without the `@` the
+* action schemas demand.
+*/
+function compactSnapshot(raw, platform) {
+	const screen = parseScreen(raw, platform);
+	const listing = renderScreen(screen);
+	return screen.truncated ? `${listing}\n  ... the tree is truncated, so some nodes are missing` : listing;
+}
+/**
+* What the model needs off an action it just ran: whether it landed, on what,
+* and whether the screen went quiet. Upstream also returns the settle diff, the
+* evidence paths, the resolution, and the cost, which are most of the step's
+* tokens and none of its meaning.
+*/
+function compactResult(name, output) {
+	if (DEVICE_TOOLS.get(name)?.output !== "outcome") return output;
+	if (typeof output !== "object" || output === null) return output;
+	const settle = asObject(output)["settle"];
+	return {
+		...pick(output, "message"),
+		...pick(output, "targetKind"),
+		...typeof settle === "object" && settle !== null ? { settle: {
+			...pick(settle, "settled"),
+			...pick(settle, "waitedMs")
+		} } : {}
+	};
+}
+/**
+* Upstream's own snapshot nodes carry a bare `e4` while the action schemas
+* demand `@e4`, and a model that sends the bare form gets an error it tends to
+* read as "refs do not work here" before falling back to coordinates for the
+* rest of the run. The listing above now prints the `@`, and this catches the
+* model that typed it from memory anyway.
+*/
+function withRefSigil(input) {
+	const target = asObject(asObject(input)["target"]);
+	if (target["kind"] !== "ref") return input;
+	const ref = target["ref"];
+	if (typeof ref !== "string" || ref.startsWith("@")) return input;
+	return {
+		...asObject(input),
+		target: {
+			...target,
+			ref: `@${ref}`
+		}
+	};
+}
+function asObject(input) {
+	return typeof input === "object" && input !== null ? input : {};
+}
+function asSnapshot(output) {
+	if (Array.isArray(asObject(output)["nodes"])) return output;
+	throw new Error("agent-device returned a snapshot without nodes");
+}
+function pick(source, key) {
+	const value = asObject(source)[key];
+	return value === void 0 ? {} : { [key]: value };
+}
+function prune(schema) {
+	if (schema.properties === void 0) return schema;
+	const properties = {};
+	for (const [key, value] of Object.entries(schema.properties)) if (!CUT_KEYS.has(key) && !(key === "target" && isDeviceAlias(value))) properties[key] = value;
+	return {
+		...schema,
+		properties,
+		...schema.required === void 0 ? {} : { required: schema.required.filter((key) => key in properties) }
+	};
+}
+/**
+* Upstream spends `target` twice. On `press`, `fill`, and `get` it is the UI
+* element, a `oneOf` over ref, selector, and point. On the rest it is an alias
+* for `deviceTarget`, an enum of device forms, and a model reading both in one
+* tool set answers the second where the first was meant. The session pins the
+* device, so the alias goes and only the UI target survives.
+*/
+function isDeviceAlias(schema) {
+	return Array.isArray(asObject(schema)["enum"]);
+}
+/** A tool the table does not name still reports, by its name alone. */
+function toolRecord(name, input) {
+	const detail = DEVICE_TOOLS.get(name)?.detail ?? null;
+	const target = detail === null ? null : readText(input, detail);
+	return target === null ? {
+		kind: "tool",
+		name
+	} : {
+		kind: "tool",
+		name,
+		target
+	};
+}
+/**
+* The text a `fill` or a `type` wrote, for the nested step under it. Never
+* masked: the model composed this string and already holds it, so hiding it
+* from the report would cost evidence and buy nothing. A credential belongs in
+* a deterministic `fill(text, { secret: true })` outside `act`.
+*/
+function typedText(name, input) {
+	return DEVICE_TOOLS.get(name)?.echoesText === true ? readText(input, "text") : null;
+}
+function readText(input, key) {
+	const value = asObject(input)[key];
+	return typeof value === "string" && value.length > 0 ? value : null;
+}
+//#endregion
+//#region src/ai/act.ts
+const TRANSCRIPT_RESULT_LIMIT = 2048;
+function instructionsFor(platform) {
+	return [
+		`You are driving a ${platform} mobile app that is already launched and in the foreground.`,
+		"Start with snapshot.",
+		"A snapshot prints one node per line, indented by depth, as in: @e4 [button] \"Sign in\" #signIn",
+		"Use press to tap a node and fill to replace a field's text.",
+		"Both take the ref as { \"kind\": \"ref\", \"ref\": \"@e4\" }, copied exactly as the line printed it.",
+		"Every ref stops working when the next command runs, so snapshot again after each action before you use one.",
+		"Use coordinates only when no line on the snapshot is the thing you need.",
+		"Once a snapshot shows the instruction is satisfied, call done with outcome \"completed\" and a one-line summary. Do not take a second snapshot to double-check.",
+		"If you cannot proceed, call done with outcome \"blocked\" and say what stopped you."
+	].join("\n");
+}
+const EXTRACT_INSTRUCTIONS = [
+	"You are reading one accessibility tree captured from a mobile app.",
+	"Each line is one node, indented by depth, carrying its ref, role, name, and test id, as in: @e4 [button] \"Sign in\" #signIn",
+	"The tree never prints a field's value.",
+	"Answer from what the tree shows, not from what the app is expected to show."
+].join("\n");
+/**
+* Runs the model against the tools until it calls `done`. Nothing here knows
+* about a session, so a test drives it with fake tools and a mock model.
+*
+* A tool error thrown inside `execute` is not caught: the AI SDK hands it back
+* to the model as a tool result, which is what lets it recover from a ref that
+* went stale by taking a fresh snapshot.
+*/
+function runAct(run) {
+	return run.sink.step(renderTitle({
+		kind: "act",
+		instruction: run.instruction
+	}), async () => {
+		const { ToolLoopAgent, hasToolCall, jsonSchema, stepCountIs, tool } = await loadAi();
+		const result = await new ToolLoopAgent({
+			model: languageModel(run.model),
+			instructions: instructionsFor(run.platform),
+			tools: {
+				...reporting(run.tools, run.sink),
+				done: tool({
+					description: "Finish the instruction. Call this when it is satisfied, or when you cannot proceed.",
+					inputSchema: jsonSchema({
+						type: "object",
+						properties: {
+							outcome: {
+								type: "string",
+								enum: ["completed", "blocked"],
+								description: "completed when the instruction is satisfied, blocked when it is not"
+							},
+							summary: {
+								type: "string",
+								description: "One line saying what you did, or what stopped you."
+							}
+						},
+						required: ["outcome", "summary"],
+						additionalProperties: false
+					})
+				})
+			},
+			stopWhen: [hasToolCall("done"), stepCountIs(run.maxSteps)]
+		}).generate({
+			prompt: run.instruction,
+			abortSignal: AbortSignal.timeout(run.timeout)
+		}).catch(async (error) => {
+			if (!timedOut(error)) throw error;
+			throw new TouchpressError({
+				kind: "ai-timeout",
+				instruction: run.instruction,
+				timeoutMs: run.timeout,
+				screen: await run.screen()
+			});
+		});
+		await run.sink.attach({
+			name: `ai-act-${String(run.attempt)}.json`,
+			contentType: "application/json",
+			body: JSON.stringify({
+				instruction: run.instruction,
+				usage: result.usage,
+				steps: result.steps.map((step) => {
+					const errors = toolErrors(step.content);
+					return {
+						text: step.text,
+						toolCalls: step.toolCalls.map((call) => ({
+							name: call.toolName,
+							input: call.input
+						})),
+						toolResults: step.toolResults.map((toolResult) => ({
+							name: toolResult.toolName,
+							output: clip(toolResult.output)
+						})),
+						...errors.length === 0 ? {} : { toolErrors: errors }
+					};
+				})
+			}, null, 2)
+		});
+		const outcome = outcomeOf(result.steps.at(-1)?.toolCalls);
+		if (outcome === null) throw new TouchpressError({
+			kind: "ai-incomplete",
+			instruction: run.instruction,
+			steps: result.steps.length,
+			screen: await run.screen()
+		});
+		if (outcome.kind === "blocked") throw new TouchpressError({
+			kind: "ai-blocked",
+			instruction: run.instruction,
+			summary: outcome.summary,
+			screen: await run.screen()
+		});
+		return outcome.summary;
+	});
+}
+/**
+* The public types are structural so the main entry's declarations never
+* import from `ai`, and these two casts are where they meet the SDK's own.
+* Every `LanguageModel` and every `FlexibleSchema` satisfies the structural
+* type it is cast from, so nothing the SDK accepts is turned away.
+*/
+function languageModel(model) {
+	return model;
+}
+function flexibleSchema(schema) {
+	return schema;
+}
+/** One capture, one question, one answer. No tools, so the model cannot change the screen it is describing. */
+function runExtract(run) {
+	return run.sink.step(renderTitle({
+		kind: "extract",
+		question: run.question
+	}), async () => {
+		const { ToolLoopAgent, Output } = await loadAi();
+		return (await new ToolLoopAgent({
+			model: languageModel(run.model),
+			instructions: EXTRACT_INSTRUCTIONS,
+			output: Output.object({ schema: flexibleSchema(run.schema) })
+		}).generate({
+			prompt: [
+				`Question: ${run.question}`,
+				``,
+				`Screen:`,
+				run.screen
+			].join("\n"),
+			abortSignal: AbortSignal.timeout(run.timeout)
+		})).output;
+	});
+}
+/**
+* Every model action becomes a step wrapping its own execution, so a report
+* shows the loop as it ran. The executions share one queue, because the AI SDK
+* runs the tool calls of one step concurrently and a snapshot overlapping a
+* press on the device reads a screen the press is changing.
+*/
+function reporting(tools, sink) {
+	const queue = createQueue();
+	const wrapped = {};
+	for (const [name, built] of Object.entries(tools)) {
+		const { execute } = built;
+		if (execute === void 0) {
+			wrapped[name] = built;
+			continue;
+		}
+		wrapped[name] = {
+			...built,
+			execute: (input, options) => queue.enqueue(() => sink.step(renderTitle(toolRecord(name, input)), async () => {
+				const text = typedText(name, input);
+				if (text !== null) await sink.step(renderTitle({
+					kind: "typed",
+					typed: {
+						kind: "text",
+						value: text
+					}
+				}), () => Promise.resolve(), { box: true });
+				return execute(input, options);
+			}))
+		};
+	}
+	return wrapped;
+}
+/**
+* The `done` call is validated here rather than trusted, because a JSON schema
+* handed to `jsonSchema()` describes the tool to the model and validates
+* nothing. A malformed call is a run that never reached an outcome.
+*/
+function outcomeOf(calls) {
+	const call = calls?.find((one) => one.toolName === "done");
+	if (call === void 0 || typeof call.input !== "object" || call.input === null) return null;
+	const summary = Reflect.get(call.input, "summary");
+	if (typeof summary !== "string") return null;
+	const outcome = Reflect.get(call.input, "outcome");
+	if (outcome === "completed") return {
+		kind: "completed",
+		summary
+	};
+	if (outcome === "blocked") return {
+		kind: "blocked",
+		summary
+	};
+	return null;
+}
+/** `AbortSignal.timeout` rejects with a DOMException named TimeoutError, which a provider surfaces as is or as an AbortError. */
+function timedOut(error) {
+	return error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError");
+}
+/**
+* The calls that failed. `step.toolResults` holds only the ones that returned,
+* so a transcript built from it alone shows a loop doing nothing and never says
+* why, which is what a nine-error run looked like from the attachment.
+*/
+function toolErrors(content) {
+	return content.filter((part) => part.type === "tool-error").map((part) => ({
+		name: part.toolName,
+		input: part.input,
+		error: messageOf(part.error)
+	}));
+}
+function messageOf(error) {
+	if (error instanceof Error) return error.message;
+	return typeof error === "string" ? error : JSON.stringify(error) ?? "unknown error";
+}
+function clip(output) {
+	const text = JSON.stringify(output) ?? "undefined";
+	return text.length <= TRANSCRIPT_RESULT_LIMIT ? text : `${text.slice(0, TRANSCRIPT_RESULT_LIMIT)}...`;
+}
+//#endregion
+//#region src/ai/device.ts
+const DEFAULT_ACT_TIMEOUT_MS = 12e4;
+const DEFAULT_ACT_STEPS = 25;
+const DEFAULT_EXTRACT_TIMEOUT_MS = 6e4;
+/**
+* Adds `act` and `extract` to a device without touching what is already there.
+*
+* Both run inside `session.run`, so the whole loop holds the session queue. A
+* test body is sequential anyway, and the model's snapshot refs carry the same
+* rule every deterministic action does: they are valid for the next command
+* only, so nothing else may reach the device in between.
+*/
+function withAi(device, session, sink, model) {
+	let acts = 0;
+	let built = null;
+	const tools = () => built ??= createDeviceTools(session.name, session.options.platform);
+	return {
+		...device,
+		act: async (instruction, options) => {
+			const configured = configuredModel(model);
+			const deviceTools = await tools();
+			return session.run((one) => runAct({
+				model: configured,
+				tools: deviceTools,
+				sink,
+				instruction,
+				platform: session.options.platform,
+				maxSteps: options?.maxSteps ?? DEFAULT_ACT_STEPS,
+				timeout: options?.timeout ?? DEFAULT_ACT_TIMEOUT_MS,
+				screen: async () => renderScreen(await one.capture()),
+				attempt: acts += 1
+			}));
+		},
+		extract: async (question, schema, options) => {
+			const configured = configuredModel(model);
+			return session.run(async (one) => runExtract({
+				model: configured,
+				screen: renderScreen(await one.capture()),
+				question,
+				schema,
+				sink,
+				timeout: options?.timeout ?? DEFAULT_EXTRACT_TIMEOUT_MS
+			}));
+		}
+	};
+}
+/** Checked before the queue, so a project that forgot the key fails at once rather than holding the session. */
+function configuredModel(model) {
+	if (model === void 0) throw new TouchpressError({ kind: "ai-not-configured" });
+	return model;
+}
+//#endregion
 //#region src/playwright/fixtures.ts
 const SESSION_FIXTURE_TIMEOUT_MS = 18e4;
 const DEVICE_FIXTURE_TIMEOUT_MS = 12e4;
@@ -62,6 +579,10 @@ const setupTest = test$1.extend({
 	sessionPrefix: [TOUCHPRESS_DEFAULTS.sessionPrefix, {
 		option: true,
 		scope: "worker"
+	}],
+	aiModel: [void 0, {
+		option: true,
+		scope: "worker"
 	}]
 });
 /** The worker session already opened the app with a relaunch, so the first test skips one. */
@@ -105,11 +626,11 @@ const test = setupTest.extend({
 		scope: "worker",
 		timeout: SESSION_FIXTURE_TIMEOUT_MS
 	}],
-	device: [async ({ session }, use, testInfo) => {
+	device: [async ({ session, aiModel }, use, testInfo) => {
 		const sink = playwrightSink();
 		if (session.options.relaunch === "per-test" && startedTests.has(session)) await session.relaunch(sink);
 		startedTests.add(session);
-		await use(createDevice(session, sink));
+		await use(withAi(createDevice(session, sink), session, sink, aiModel));
 		if (shouldCapture(testInfo, session.options.evidence)) await captureEvidence(session, sink);
 	}, {
 		auto: true,
@@ -373,10 +894,16 @@ function write(path, png) {
 	mkdirSync(dirname(path), { recursive: true });
 	writeFileSync(path, png);
 }
+/**
+* The describe path is part of the name, the way Playwright's own screenshot
+* assertion names its baselines, so two blocks each holding a test called
+* "shot" do not write over one baseline. The first element is the file, which
+* `snapshotPath` already places the baseline under.
+*/
 function defaultName(info) {
 	const next = (ordinals.get(info) ?? 0) + 1;
 	ordinals.set(info, next);
-	return `${info.title.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "")}-${String(next)}.png`;
+	return `${info.titlePath.slice(1).join(" ").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "")}-${String(next)}.png`;
 }
 //#endregion
 //#region src/playwright/expect.ts

package/dist/{preflight-CCKYplYi.d.mts → preflight-Czhj9QFM.d.mts}

@@ -448,6 +448,18 @@ type ActionRecord = {
 } | {
   readonly kind: 'screenshot';
   readonly path: string;
+} | {
+  readonly kind: 'act';
+  readonly instruction: string;
+} | {
+  readonly kind: 'extract';
+  readonly question: string;
+} |
+/** One command the model ran inside an `act`. `target` is its snapshot ref, or a scroll's direction. */
+{
+  readonly kind: 'tool';
+  readonly name: string;
+  readonly target?: string;
 };
 /** A boxed step reports as one line rather than as something to open. */
 type StepOptions = {
@@ -825,6 +837,26 @@ type ErrorInfo = {
   readonly kind: 'driver';
   readonly command: string;
   readonly failure: DeviceFailure;
+} | {
+  readonly kind: 'ai-not-configured';
+} | {
+  readonly kind: 'ai-missing-peer';
+} | {
+  readonly kind: 'ai-blocked';
+  readonly instruction: string;
+  /** The model's own account of what stopped it. */
+  readonly summary: string;
+  readonly screen: string;
+} | {
+  readonly kind: 'ai-incomplete';
+  readonly instruction: string;
+  readonly steps: number;
+  readonly screen: string;
+} | {
+  readonly kind: 'ai-timeout';
+  readonly instruction: string;
+  readonly timeoutMs: number;
+  readonly screen: string;
 };
 declare class TouchpressError extends Error {
   readonly info: ErrorInfo;

package/dist/{preflight-DzSceNMx.mjs → preflight-aUba9u1k.mjs}

@@ -59,6 +59,36 @@ function formatError(info) {
 			info.screen
 		].join("\n");
 		case "driver": return `${info.command} failed: ${describeFailure(info.failure)}`;
+		case "ai-not-configured": return ["device.act and device.extract need a model.", "Set use.aiModel to a gateway model id, such as 'anthropic/claude-sonnet-5', or to a provider model instance."].join("\n");
+		case "ai-missing-peer": return ["device.act and device.extract need the optional peer dependency 'ai'.", "Install it with: pnpm add -D ai"].join("\n");
+		case "ai-blocked": return [
+			`act stopped without finishing: ${info.summary}`,
+			``,
+			`Instruction: ${info.instruction}`,
+			``,
+			`Screen:`,
+			info.screen
+		].join("\n");
+		case "ai-incomplete": return [
+			`act ran ${String(info.steps)} steps without reaching an outcome.`,
+			``,
+			`Instruction: ${info.instruction}`,
+			``,
+			"Raise maxSteps, or split the instruction into smaller ones.",
+			``,
+			`Screen:`,
+			info.screen
+		].join("\n");
+		case "ai-timeout": return [
+			`act ran out of its ${String(info.timeoutMs)}ms budget before reaching an outcome.`,
+			``,
+			`Instruction: ${info.instruction}`,
+			``,
+			"Raise the act timeout, and the test timeout with it.",
+			``,
+			`Screen:`,
+			info.screen
+		].join("\n");
 		default: throw new Error(`unhandled error info ${JSON.stringify(info)}`);
 	}
 }
@@ -115,7 +145,9 @@ function matchesText(match, candidate) {
 	switch (match.kind) {
 		case "exact": return candidate === match.value;
 		case "substring": return candidate.toLowerCase().includes(match.value.toLowerCase());
-		case "regex": return match.value.test(candidate);
+		case "regex":
+			match.value.lastIndex = 0;
+			return match.value.test(candidate);
 		default: throw new Error(`unhandled text match ${JSON.stringify(match)}`);
 	}
 }
@@ -355,89 +387,6 @@ function fail(field, detail) {
 	});
 }
 //#endregion
-//#region src/core/checks.ts
-/**
-* A `many` outcome never passes anything but `toHaveCount`. An ambiguous locator
-* is a strictness violation, and it reports through the same message path as a
-* plain mismatch rather than guessing which node was meant.
-*/
-function evaluate(check, resolution) {
-	if (check.name === "toHaveCount") {
-		const count = countOf(resolution);
-		return {
-			pass: count === check.expected,
-			actual: String(count)
-		};
-	}
-	if (resolution.outcome === "none") return {
-		pass: false,
-		actual: null
-	};
-	if (resolution.outcome === "many") return {
-		pass: false,
-		actual: `${String(resolution.nodes.length)} matching nodes`
-	};
-	const node = resolution.node;
-	switch (check.name) {
-		case "toBeVisible": return {
-			pass: true,
-			actual: describeNode(node)
-		};
-		case "toHaveText": {
-			const text = node.name ?? node.value;
-			return {
-				pass: matchesText(check.expected, text),
-				actual: text === null ? null : `"${text}"`
-			};
-		}
-		case "toHaveValue": return {
-			pass: matchesText(check.expected, node.value),
-			actual: node.value === null ? null : `"${node.value}"`
-		};
-		case "toBeEnabled": return {
-			pass: node.enabled,
-			actual: node.enabled ? "enabled" : "disabled"
-		};
-		case "toBeSelected": return {
-			pass: node.selected,
-			actual: node.selected ? "selected" : "not selected"
-		};
-		case "toBeFocused": return {
-			pass: node.focused,
-			actual: node.focused ? "focused" : "not focused"
-		};
-		default: throw new Error(`unhandled check ${JSON.stringify(check)}`);
-	}
-}
-function countOf(resolution) {
-	switch (resolution.outcome) {
-		case "one": return 1;
-		case "none": return 0;
-		case "many": return resolution.nodes.length;
-		default: throw new Error(`unhandled resolution ${JSON.stringify(resolution)}`);
-	}
-}
-/** The `Expected:` line. */
-function describeCheck(check) {
-	switch (check.name) {
-		case "toBeVisible": return "visible";
-		case "toHaveText": return `text ${describeExpected(check.expected)}`;
-		case "toHaveValue": return `value ${describeExpected(check.expected)}`;
-		case "toBeEnabled": return "enabled";
-		case "toBeSelected": return "selected";
-		case "toBeFocused": return "focused";
-		case "toHaveCount": return `count ${String(check.expected)}`;
-		default: throw new Error(`unhandled check ${JSON.stringify(check)}`);
-	}
-}
-function describeExpected(match) {
-	return match.kind === "regex" ? String(match.value) : `"${match.value}"`;
-}
-function describeNode(node) {
-	const name = node.name === null ? "" : ` "${node.name}"`;
-	return `${node.ref} [${node.role}]${name}`;
-}
-//#endregion
 //#region src/core/screen.ts
 const IOS_ROLES = {
 	Application: "application",
@@ -716,6 +665,7 @@ function renderNode(node) {
 //#endregion
 //#region src/core/report.ts
 const FILL_TEXT_LIMIT = 40;
+const PROMPT_LIMIT = 80;
 /** Rendered here rather than in an adapter so every runner produces the same text. */
 function renderTitle(record) {
 	switch (record.kind) {
@@ -729,18 +679,21 @@ function renderTitle(record) {
 		case "relaunch": return `relaunch ${record.app}`;
 		case "dismiss-overlay": return "dismiss the React Native dev overlay";
 		case "screenshot": return `screenshot ${record.path}`;
+		case "act": return `act "${truncate(record.instruction, PROMPT_LIMIT)}"`;
+		case "extract": return `extract "${truncate(record.question, PROMPT_LIMIT)}"`;
+		case "tool": return record.target === void 0 ? record.name : `${record.name} ${record.target}`;
 		default: throw new Error(`unhandled action record ${JSON.stringify(record)}`);
 	}
 }
 function renderTyped(typed) {
 	switch (typed.kind) {
-		case "text": return `type "${truncate(typed.value)}"`;
+		case "text": return `type "${truncate(typed.value, FILL_TEXT_LIMIT)}"`;
 		case "hidden": return `type ${String(typed.length)} characters`;
 		default: throw new Error(`unhandled typed value ${JSON.stringify(typed)}`);
 	}
 }
-function truncate(text) {
-	return text.length <= FILL_TEXT_LIMIT ? text : `${text.slice(0, FILL_TEXT_LIMIT)}...`;
+function truncate(text, limit) {
+	return text.length <= limit ? text : `${text.slice(0, limit)}...`;
 }
 /** Discards everything. The default for scripts, unit tests, and runners with no reporting. */
 const silentSink = {
@@ -952,6 +905,89 @@ function sleep(ms) {
 	return new Promise((done) => setTimeout(done, ms));
 }
 //#endregion
+//#region src/core/checks.ts
+/**
+* A `many` outcome never passes anything but `toHaveCount`. An ambiguous locator
+* is a strictness violation, and it reports through the same message path as a
+* plain mismatch rather than guessing which node was meant.
+*/
+function evaluate(check, resolution) {
+	if (check.name === "toHaveCount") {
+		const count = countOf(resolution);
+		return {
+			pass: count === check.expected,
+			actual: String(count)
+		};
+	}
+	if (resolution.outcome === "none") return {
+		pass: false,
+		actual: null
+	};
+	if (resolution.outcome === "many") return {
+		pass: false,
+		actual: `${String(resolution.nodes.length)} matching nodes`
+	};
+	const node = resolution.node;
+	switch (check.name) {
+		case "toBeVisible": return {
+			pass: true,
+			actual: describeNode(node)
+		};
+		case "toHaveText": {
+			const text = node.name ?? node.value;
+			return {
+				pass: matchesText(check.expected, text),
+				actual: text === null ? null : `"${text}"`
+			};
+		}
+		case "toHaveValue": return {
+			pass: matchesText(check.expected, node.value),
+			actual: node.value === null ? null : `"${node.value}"`
+		};
+		case "toBeEnabled": return {
+			pass: node.enabled,
+			actual: node.enabled ? "enabled" : "disabled"
+		};
+		case "toBeSelected": return {
+			pass: node.selected,
+			actual: node.selected ? "selected" : "not selected"
+		};
+		case "toBeFocused": return {
+			pass: node.focused,
+			actual: node.focused ? "focused" : "not focused"
+		};
+		default: throw new Error(`unhandled check ${JSON.stringify(check)}`);
+	}
+}
+function countOf(resolution) {
+	switch (resolution.outcome) {
+		case "one": return 1;
+		case "none": return 0;
+		case "many": return resolution.nodes.length;
+		default: throw new Error(`unhandled resolution ${JSON.stringify(resolution)}`);
+	}
+}
+/** The `Expected:` line. */
+function describeCheck(check) {
+	switch (check.name) {
+		case "toBeVisible": return "visible";
+		case "toHaveText": return `text ${describeExpected(check.expected)}`;
+		case "toHaveValue": return `value ${describeExpected(check.expected)}`;
+		case "toBeEnabled": return "enabled";
+		case "toBeSelected": return "selected";
+		case "toBeFocused": return "focused";
+		case "toHaveCount": return `count ${String(check.expected)}`;
+		default: throw new Error(`unhandled check ${JSON.stringify(check)}`);
+	}
+}
+function describeExpected(match) {
+	return match.kind === "regex" ? String(match.value) : `"${match.value}"`;
+}
+function describeNode(node) {
+	const name = node.name === null ? "" : ` "${node.name}"`;
+	return `${node.ref} [${node.role}]${name}`;
+}
+//#endregion
 //#region src/core/probe.ts
 const POLL_INTERVAL_MS = 250;
 const SCREEN_LISTING_NODES = 60;
@@ -1266,10 +1302,10 @@ function perform(session, sink, record, options, dispatch, write) {
 					}
 					attempts += 1;
 					if (!settled.settled) sink.note("settle", `${renderTitle(record)} finished before the screen went quiet`);
-					if (confirmation === null) return;
+					if (confirmation === null || write === void 0) return;
 					if (attempts === 1) await sink.step(renderTitle({
 						kind: "typed",
-						typed: typedOf(confirmation)
+						typed: typedOf(resolution.node.role, write)
 					}), () => Promise.resolve(), { box: true });
 					target = identityOf(screen, resolution.node);
 					screen = await device.capture();
@@ -1361,12 +1397,13 @@ function confirmationOf(role, write) {
 		kind: "mask",
 		length: write.text.length
 	};
+	const value = normalizeText(write.text);
 	return write.secret ? {
 		kind: "secret",
-		value: write.text
+		value
 	} : {
 		kind: "open",
-		value: write.text
+		value
 	};
 }
 function holds(confirmation, actual) {
@@ -1394,22 +1431,16 @@ function expectedOf(confirmation) {
 		default: throw new Error(`unhandled confirmation ${JSON.stringify(confirmation)}`);
 	}
 }
-function typedOf(confirmation) {
-	switch (confirmation.kind) {
-		case "mask": return {
-			kind: "hidden",
-			length: confirmation.length
-		};
-		case "secret": return {
-			kind: "hidden",
-			length: confirmation.value.length
-		};
-		case "open": return {
-			kind: "text",
-			value: confirmation.value
-		};
-		default: throw new Error(`unhandled confirmation ${JSON.stringify(confirmation)}`);
-	}
+/** What the report says was typed, verbatim, which the normalized confirmation no longer holds. */
+function typedOf(role, write) {
+	if (role === "secure-text-field" || write.secret) return {
+		kind: "hidden",
+		length: write.text.length
+	};
+	return {
+		kind: "text",
+		value: write.text
+	};
 }
 /** An actual value may be repeated back only as far as the expected one could be. */
 function disclose(expected, actual) {
@@ -1713,21 +1744,24 @@ function compareScreenshot(expected, actual, options) {
 	};
 }
 /**
-* The box is clamped to the image, because a rect comes from a snapshot and a
+* The box is clipped to the image, because a rect comes from a snapshot and a
 * screenshot is a separate capture. A control flush against the bottom edge can
-* round a pixel past it, and that is not a reason to fail an assertion.
+* round a pixel past it, and that is not a reason to fail an assertion. A box
+* with no pixel inside the image is a different thing, a rect the screenshot
+* does not show, and cutting a placeholder out of the corner would hide that.
 */
 function cropScreenshot(source, box) {
 	const image = PNG.sync.read(source);
-	const clamped = clamp(box, {
+	const clipped = intersect(box, {
 		width: image.width,
 		height: image.height
 	});
+	if (clipped === null) throw new Error(`the crop at ${String(box.x)},${String(box.y)} ${String(box.width)}x${String(box.height)} lies outside the ${String(image.width)}x${String(image.height)} screenshot`);
 	const cut = new PNG({
-		width: clamped.width,
-		height: clamped.height
+		width: clipped.width,
+		height: clipped.height
 	});
-	PNG.bitblt(image, cut, clamped.x, clamped.y, clamped.width, clamped.height, 0, 0);
+	PNG.bitblt(image, cut, clipped.x, clipped.y, clipped.width, clipped.height, 0, 0);
 	return PNG.sync.write(cut);
 }
 function sizeOf(source) {
@@ -1756,21 +1790,30 @@ function relativeTo(box, origin) {
 		y: box.y - origin.y
 	};
 }
-function clamp(box, size) {
-	const x = Math.min(Math.max(box.x, 0), Math.max(size.width - 1, 0));
-	const y = Math.min(Math.max(box.y, 0), Math.max(size.height - 1, 0));
+/**
+* The part of the box inside the image, or null when none of it is. A mask
+* resolved off a node outside a crop must paint nothing, because moving it
+* inside would black out real pixels at the crop's edge.
+*/
+function intersect(box, size) {
+	const left = Math.max(box.x, 0);
+	const top = Math.max(box.y, 0);
+	const right = Math.min(box.x + box.width, size.width);
+	const bottom = Math.min(box.y + box.height, size.height);
+	if (right <= left || bottom <= top) return null;
 	return {
-		x,
-		y,
-		width: Math.max(Math.min(box.width, size.width - x), 1),
-		height: Math.max(Math.min(box.height, size.height - y), 1)
+		x: left,
+		y: top,
+		width: right - left,
+		height: bottom - top
 	};
 }
 function paintBlack(image, box) {
-	const region = clamp(box, {
+	const region = intersect(box, {
 		width: image.width,
 		height: image.height
 	});
+	if (region === null) return;
 	for (let row = region.y; row < region.y + region.height; row += 1) for (let column = region.x; column < region.x + region.width; column += 1) {
 		const at = image.width * row + column << 2;
 		image.data[at] = 0;
@@ -1862,4 +1905,4 @@ function notBooted(platform, name, booted) {
 	return `No booted ${platform} device is named '${name}'. Booted right now: ${booted.map((device) => `'${device.name}'`).join(", ")}. Set use.deviceName to one of those or boot '${name}'.`;
 }
 //#endregion
-export { textMatch as A, describeCheck as C, parseDeviceOptions as D, deviceNameForSlot as E, describeQuery as O, resolve as S, TOUCHPRESS_DEFAULTS as T, sleep as _, sizeOf as a, parseScreen as b, createClient as c, createScrollSearch as d, directionToward as f, sessionName as g, openSession as h, relativeTo as i, TouchpressError as j, normalizeText as k, captureEvidence as l, probe as m, compareScreenshot as n, toPixelBox as o, formatFailure as p, cropScreenshot as r, createAgentDeviceDriver as s, preflight as t, createDevice as u, renderTitle as v, evaluate as w, renderScreen as x, silentSink as y };
+export { normalizeText as A, parseScreen as C, deviceNameForSlot as D, TOUCHPRESS_DEFAULTS as E, TouchpressError as M, parseDeviceOptions as O, silentSink as S, resolve as T, createQueue as _, sizeOf as a, sleep as b, createClient as c, createScrollSearch as d, directionToward as f, evaluate as g, describeCheck as h, relativeTo as i, textMatch as j, describeQuery as k, captureEvidence as l, probe as m, compareScreenshot as n, toPixelBox as o, formatFailure as p, cropScreenshot as r, createAgentDeviceDriver as s, preflight as t, createDevice as u, openSession as v, renderScreen as w, renderTitle as x, sessionName as y };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "touchpress",
-  "version": "0.0.2",
+  "version": "0.1.1",
   "description": "End-to-end testing for mobile apps.",
   "homepage": "https://github.com/wobsoriano/touchpress#readme",
   "bugs": {
@@ -33,13 +33,21 @@
     "@playwright/test": "1.63.0",
     "@types/node": "^26.1.1",
     "@types/pngjs": "^6.0.5",
+    "ai": "^7.0.93",
     "bumpp": "^11.1.0",
     "typescript": "^7.0.2",
     "vite": "npm:@voidzero-dev/vite-plus-core@0.3.0",
-    "vite-plus": "0.3.0"
+    "vite-plus": "0.3.0",
+    "zod": "^4.5.4"
   },
   "peerDependencies": {
-    "@playwright/test": ">=1.63"
+    "@playwright/test": ">=1.63",
+    "ai": "^6.0.0 || ^7.0.0"
+  },
+  "peerDependenciesMeta": {
+    "ai": {
+      "optional": true
+    }
   },
   "devEngines": {
     "packageManager": {