touchpress 0.0.1 → 0.1.0

package/dist/index.mjs

@@ -1,12 +1,505 @@
-import { A as textMatch, D as parseDeviceOptions, S as resolve, T as TANGERE_DEFAULTS, _ as sleep, a as sizeOf, c as createClient, h as openSession, i as relativeTo, j as TangereError, l as captureEvidence, n as compareScreenshot, o as toPixelBox, r as cropScreenshot, s as createAgentDeviceDriver, t as preflight, u as createDevice, y as silentSink } from "./preflight-C83jCNPs.mjs";
+import { A as textMatch, C as renderScreen, D as parseDeviceOptions, S as parseScreen, T as TOUCHPRESS_DEFAULTS, _ as sleep, a as sizeOf, b as renderTitle, c as createClient, h as openSession, i as relativeTo, j as TouchpressError, l as captureEvidence, n as compareScreenshot, o as toPixelBox, r as cropScreenshot, s as createAgentDeviceDriver, t as preflight, u as createDevice, w as resolve, x as silentSink } from "./preflight-B7KrbqSO.mjs";
 import { expect as expect$1, test as test$1 } from "@playwright/test";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname } from "node:path";
+//#region src/ai/tools.ts
+/**
+* The AI SDK is an optional peer, so it is imported here and never at module
+* scope. Importing `touchpress` must not require it, because a project that
+* calls neither `act` nor `extract` never installs it.
+*/
+async function loadAi() {
+	try {
+		return await import("ai");
+	} catch {
+		throw new TouchpressError({ kind: "ai-missing-peer" });
+	}
+}
+const READ = {
+	detail: null,
+	echoesText: false,
+	output: "raw"
+};
+const DEVICE_TOOLS = /* @__PURE__ */ new Map([
+	["snapshot", {
+		detail: null,
+		echoesText: false,
+		output: "screen"
+	}],
+	["press", {
+		detail: "target",
+		echoesText: false,
+		output: "outcome"
+	}],
+	["fill", {
+		detail: "target",
+		echoesText: true,
+		output: "outcome"
+	}],
+	["type", {
+		detail: null,
+		echoesText: true,
+		output: "outcome"
+	}],
+	["scroll", {
+		detail: "direction",
+		echoesText: false,
+		output: "outcome"
+	}],
+	["back", {
+		detail: null,
+		echoesText: false,
+		output: "outcome"
+	}],
+	["wait", READ],
+	["get", READ],
+	["is", READ],
+	["alert", READ]
+]);
+/**
+* Everything an input schema says about which device, daemon, or workspace a
+* command reaches. The session already pins all of it, and a model that could
+* name a daemon could leave the device this test opened.
+*
+* `recordAs` rides along for a different reason: a `fill` carrying it fails
+* unless script recording is armed, which under touchpress it never is.
+*/
+const CUT_KEYS = /* @__PURE__ */ new Set([
+	"udid",
+	"serial",
+	"device",
+	"deviceTarget",
+	"daemonBaseUrl",
+	"daemonAuthToken",
+	"tenant",
+	"runId",
+	"leaseId",
+	"cwd",
+	"debug",
+	"iosSimulatorDeviceSet",
+	"iosXctestrunFile",
+	"iosXctestDerivedDataPath",
+	"iosXctestEnvDir",
+	"androidDeviceAllowlist",
+	"noRecord",
+	"record",
+	"saveScript",
+	"stateDir",
+	"recordAs"
+]);
+/**
+* The tools a model drives the app with, built from agent-device's own command
+* registry so the descriptions and the executors stay upstream's. Three things
+* change: the set is narrowed to the ten commands that perceive and act, every
+* key that could point a command at another device is cut from the input
+* schema, and each executor is wrapped so what crosses to the model is the
+* shape the model can act on. Building them contacts no device.
+*/
+async function createDeviceTools(session, platform) {
+	const { createAgentDeviceTools } = await import("agent-device/ai-sdk");
+	const { jsonSchema, tool } = await loadAi();
+	const { tools } = await createAgentDeviceTools({
+		session,
+		platform
+	});
+	const kept = {};
+	for (const name of DEVICE_TOOLS.keys()) {
+		const built = tools[name];
+		if (built === void 0) throw new Error(`agent-device no longer exposes the "${name}" tool`);
+		kept[name] = tool({
+			description: built.description,
+			inputSchema: jsonSchema(prune(built.inputSchema.jsonSchema)),
+			execute: wrapDeviceTool(name, platform, built.execute)
+		});
+	}
+	return kept;
+}
+/**
+* The one place a command's input and output are reshaped for the model, so the
+* table above stays the only thing that says which command gets which shape.
+*/
+function wrapDeviceTool(name, platform, execute) {
+	const shape = DEVICE_TOOLS.get(name)?.output ?? "raw";
+	return async (input, options) => {
+		const output = await execute(shape === "screen" ? {
+			...asObject(input),
+			forceFull: true
+		} : withRefSigil(input), options);
+		return shape === "screen" ? compactSnapshot(asSnapshot(output), platform) : compactResult(name, output);
+	};
+}
+/**
+* The tree the model reads, in the same listing a failure message prints, one
+* node per line indented by depth. The JSON upstream returns says the same thing
+* with rects, indexes, and flags the model never uses, at three to seven times
+* the size across this repo's fixtures, and its refs arrive without the `@` the
+* action schemas demand.
+*/
+function compactSnapshot(raw, platform) {
+	const screen = parseScreen(raw, platform);
+	const listing = renderScreen(screen);
+	return screen.truncated ? `${listing}\n  ... the tree is truncated, so some nodes are missing` : listing;
+}
+/**
+* What the model needs off an action it just ran: whether it landed, on what,
+* and whether the screen went quiet. Upstream also returns the settle diff, the
+* evidence paths, the resolution, and the cost, which are most of the step's
+* tokens and none of its meaning.
+*/
+function compactResult(name, output) {
+	if (DEVICE_TOOLS.get(name)?.output !== "outcome") return output;
+	if (typeof output !== "object" || output === null) return output;
+	const settle = asObject(output)["settle"];
+	return {
+		...pick(output, "message"),
+		...pick(output, "targetKind"),
+		...typeof settle === "object" && settle !== null ? { settle: {
+			...pick(settle, "settled"),
+			...pick(settle, "waitedMs")
+		} } : {}
+	};
+}
+/**
+* Upstream's own snapshot nodes carry a bare `e4` while the action schemas
+* demand `@e4`, and a model that sends the bare form gets an error it tends to
+* read as "refs do not work here" before falling back to coordinates for the
+* rest of the run. The listing above now prints the `@`, and this catches the
+* model that typed it from memory anyway.
+*/
+function withRefSigil(input) {
+	const target = asObject(asObject(input)["target"]);
+	if (target["kind"] !== "ref") return input;
+	const ref = target["ref"];
+	if (typeof ref !== "string" || ref.startsWith("@")) return input;
+	return {
+		...asObject(input),
+		target: {
+			...target,
+			ref: `@${ref}`
+		}
+	};
+}
+function asObject(input) {
+	return typeof input === "object" && input !== null ? input : {};
+}
+function asSnapshot(output) {
+	if (Array.isArray(asObject(output)["nodes"])) return output;
+	throw new Error("agent-device returned a snapshot without nodes");
+}
+function pick(source, key) {
+	const value = asObject(source)[key];
+	return value === void 0 ? {} : { [key]: value };
+}
+function prune(schema) {
+	if (schema.properties === void 0) return schema;
+	const properties = {};
+	for (const [key, value] of Object.entries(schema.properties)) if (!CUT_KEYS.has(key) && !(key === "target" && isDeviceAlias(value))) properties[key] = value;
+	return {
+		...schema,
+		properties,
+		...schema.required === void 0 ? {} : { required: schema.required.filter((key) => key in properties) }
+	};
+}
+/**
+* Upstream spends `target` twice. On `press`, `fill`, and `get` it is the UI
+* element, a `oneOf` over ref, selector, and point. On the rest it is an alias
+* for `deviceTarget`, an enum of device forms, and a model reading both in one
+* tool set answers the second where the first was meant. The session pins the
+* device, so the alias goes and only the UI target survives.
+*/
+function isDeviceAlias(schema) {
+	return Array.isArray(asObject(schema)["enum"]);
+}
+/** A tool the table does not name still reports, by its name alone. */
+function toolRecord(name, input) {
+	const detail = DEVICE_TOOLS.get(name)?.detail ?? null;
+	const target = detail === null ? null : readText(input, detail);
+	return target === null ? {
+		kind: "tool",
+		name
+	} : {
+		kind: "tool",
+		name,
+		target
+	};
+}
+/**
+* The text a `fill` or a `type` wrote, for the nested step under it. Never
+* masked: the model composed this string and already holds it, so hiding it
+* from the report would cost evidence and buy nothing. A credential belongs in
+* a deterministic `fill(text, { secret: true })` outside `act`.
+*/
+function typedText(name, input) {
+	return DEVICE_TOOLS.get(name)?.echoesText === true ? readText(input, "text") : null;
+}
+function readText(input, key) {
+	const value = asObject(input)[key];
+	return typeof value === "string" && value.length > 0 ? value : null;
+}
+//#endregion
+//#region src/ai/act.ts
+const TRANSCRIPT_RESULT_LIMIT = 2048;
+function instructionsFor(platform) {
+	return [
+		`You are driving a ${platform} mobile app that is already launched and in the foreground.`,
+		"Start with snapshot.",
+		"A snapshot prints one node per line, indented by depth, as in: @e4 [button] \"Sign in\" #signIn",
+		"Use press to tap a node and fill to replace a field's text.",
+		"Both take the ref as { \"kind\": \"ref\", \"ref\": \"@e4\" }, copied exactly as the line printed it.",
+		"Every ref stops working when the next command runs, so snapshot again after each action before you use one.",
+		"Use coordinates only when no line on the snapshot is the thing you need.",
+		"Once a snapshot shows the instruction is satisfied, call done with outcome \"completed\" and a one-line summary. Do not take a second snapshot to double-check.",
+		"If you cannot proceed, call done with outcome \"blocked\" and say what stopped you."
+	].join("\n");
+}
+const EXTRACT_INSTRUCTIONS = [
+	"You are reading one accessibility tree captured from a mobile app.",
+	"Each line is one node, indented by depth, carrying its ref, role, name, and test id, as in: @e4 [button] \"Sign in\" #signIn",
+	"The tree never prints a field's value.",
+	"Answer from what the tree shows, not from what the app is expected to show."
+].join("\n");
+/**
+* Runs the model against the tools until it calls `done`. Nothing here knows
+* about a session, so a test drives it with fake tools and a mock model.
+*
+* A tool error thrown inside `execute` is not caught: the AI SDK hands it back
+* to the model as a tool result, which is what lets it recover from a ref that
+* went stale by taking a fresh snapshot.
+*/
+function runAct(run) {
+	return run.sink.step(renderTitle({
+		kind: "act",
+		instruction: run.instruction
+	}), async () => {
+		const { ToolLoopAgent, hasToolCall, jsonSchema, stepCountIs, tool } = await loadAi();
+		const result = await new ToolLoopAgent({
+			model: run.model,
+			instructions: instructionsFor(run.platform),
+			tools: {
+				...reporting(run.tools, run.sink),
+				done: tool({
+					description: "Finish the instruction. Call this when it is satisfied, or when you cannot proceed.",
+					inputSchema: jsonSchema({
+						type: "object",
+						properties: {
+							outcome: {
+								type: "string",
+								enum: ["completed", "blocked"],
+								description: "completed when the instruction is satisfied, blocked when it is not"
+							},
+							summary: {
+								type: "string",
+								description: "One line saying what you did, or what stopped you."
+							}
+						},
+						required: ["outcome", "summary"],
+						additionalProperties: false
+					})
+				})
+			},
+			stopWhen: [hasToolCall("done"), stepCountIs(run.maxSteps)]
+		}).generate({
+			prompt: run.instruction,
+			abortSignal: AbortSignal.timeout(run.timeout)
+		}).catch(async (error) => {
+			if (!timedOut(error)) throw error;
+			throw new TouchpressError({
+				kind: "ai-timeout",
+				instruction: run.instruction,
+				timeoutMs: run.timeout,
+				screen: await run.screen()
+			});
+		});
+		await run.sink.attach({
+			name: `ai-act-${String(run.attempt)}.json`,
+			contentType: "application/json",
+			body: JSON.stringify({
+				instruction: run.instruction,
+				usage: result.usage,
+				steps: result.steps.map((step) => {
+					const errors = toolErrors(step.content);
+					return {
+						text: step.text,
+						toolCalls: step.toolCalls.map((call) => ({
+							name: call.toolName,
+							input: call.input
+						})),
+						toolResults: step.toolResults.map((toolResult) => ({
+							name: toolResult.toolName,
+							output: clip(toolResult.output)
+						})),
+						...errors.length === 0 ? {} : { toolErrors: errors }
+					};
+				})
+			}, null, 2)
+		});
+		const outcome = outcomeOf(result.steps.at(-1)?.toolCalls);
+		if (outcome === null) throw new TouchpressError({
+			kind: "ai-incomplete",
+			instruction: run.instruction,
+			steps: result.steps.length,
+			screen: await run.screen()
+		});
+		if (outcome.kind === "blocked") throw new TouchpressError({
+			kind: "ai-blocked",
+			instruction: run.instruction,
+			summary: outcome.summary,
+			screen: await run.screen()
+		});
+		return outcome.summary;
+	});
+}
+/** One capture, one question, one answer. No tools, so the model cannot change the screen it is describing. */
+function runExtract(run) {
+	return run.sink.step(renderTitle({
+		kind: "extract",
+		question: run.question
+	}), async () => {
+		const { ToolLoopAgent, Output } = await loadAi();
+		return (await new ToolLoopAgent({
+			model: run.model,
+			instructions: EXTRACT_INSTRUCTIONS,
+			output: Output.object({ schema: run.schema })
+		}).generate({
+			prompt: [
+				`Question: ${run.question}`,
+				``,
+				`Screen:`,
+				run.screen
+			].join("\n"),
+			abortSignal: AbortSignal.timeout(run.timeout)
+		})).output;
+	});
+}
+/** Every model action becomes a step wrapping its own execution, so a report shows the loop as it ran. */
+function reporting(tools, sink) {
+	const wrapped = {};
+	for (const [name, built] of Object.entries(tools)) {
+		const { execute } = built;
+		if (execute === void 0) {
+			wrapped[name] = built;
+			continue;
+		}
+		wrapped[name] = {
+			...built,
+			execute: (input, options) => sink.step(renderTitle(toolRecord(name, input)), async () => {
+				const text = typedText(name, input);
+				if (text !== null) await sink.step(renderTitle({
+					kind: "typed",
+					typed: {
+						kind: "text",
+						value: text
+					}
+				}), () => Promise.resolve(), { box: true });
+				return execute(input, options);
+			})
+		};
+	}
+	return wrapped;
+}
+/**
+* The `done` call is validated here rather than trusted, because a JSON schema
+* handed to `jsonSchema()` describes the tool to the model and validates
+* nothing. A malformed call is a run that never reached an outcome.
+*/
+function outcomeOf(calls) {
+	const call = calls?.find((one) => one.toolName === "done");
+	if (call === void 0 || typeof call.input !== "object" || call.input === null) return null;
+	const summary = Reflect.get(call.input, "summary");
+	if (typeof summary !== "string") return null;
+	const outcome = Reflect.get(call.input, "outcome");
+	if (outcome === "completed") return {
+		kind: "completed",
+		summary
+	};
+	if (outcome === "blocked") return {
+		kind: "blocked",
+		summary
+	};
+	return null;
+}
+/** `AbortSignal.timeout` rejects with a DOMException named TimeoutError, which a provider surfaces as is or as an AbortError. */
+function timedOut(error) {
+	return error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError");
+}
+/**
+* The calls that failed. `step.toolResults` holds only the ones that returned,
+* so a transcript built from it alone shows a loop doing nothing and never says
+* why, which is what a nine-error run looked like from the attachment.
+*/
+function toolErrors(content) {
+	return content.filter((part) => part.type === "tool-error").map((part) => ({
+		name: part.toolName,
+		input: part.input,
+		error: messageOf(part.error)
+	}));
+}
+function messageOf(error) {
+	if (error instanceof Error) return error.message;
+	return typeof error === "string" ? error : JSON.stringify(error) ?? "unknown error";
+}
+function clip(output) {
+	const text = JSON.stringify(output) ?? "undefined";
+	return text.length <= TRANSCRIPT_RESULT_LIMIT ? text : `${text.slice(0, TRANSCRIPT_RESULT_LIMIT)}...`;
+}
+//#endregion
+//#region src/ai/device.ts
+const DEFAULT_ACT_TIMEOUT_MS = 12e4;
+const DEFAULT_ACT_STEPS = 25;
+const DEFAULT_EXTRACT_TIMEOUT_MS = 6e4;
+/**
+* Adds `act` and `extract` to a device without touching what is already there.
+*
+* Both run inside `session.run`, so the whole loop holds the session queue. A
+* test body is sequential anyway, and the model's snapshot refs carry the same
+* rule every deterministic action does: they are valid for the next command
+* only, so nothing else may reach the device in between.
+*/
+function withAi(device, session, sink, model) {
+	let acts = 0;
+	let built = null;
+	const tools = () => built ??= createDeviceTools(session.name, session.options.platform);
+	return {
+		...device,
+		act: async (instruction, options) => {
+			const configured = configuredModel(model);
+			const deviceTools = await tools();
+			return session.run((one) => runAct({
+				model: configured,
+				tools: deviceTools,
+				sink,
+				instruction,
+				platform: session.options.platform,
+				maxSteps: options?.maxSteps ?? DEFAULT_ACT_STEPS,
+				timeout: options?.timeout ?? DEFAULT_ACT_TIMEOUT_MS,
+				screen: async () => renderScreen(await one.capture()),
+				attempt: acts += 1
+			}));
+		},
+		extract: async (question, schema, options) => {
+			const configured = configuredModel(model);
+			return session.run(async (one) => runExtract({
+				model: configured,
+				screen: renderScreen(await one.capture()),
+				question,
+				schema,
+				sink,
+				timeout: options?.timeout ?? DEFAULT_EXTRACT_TIMEOUT_MS
+			}));
+		}
+	};
+}
+/** Checked before the queue, so a project that forgot the key fails at once rather than holding the session. */
+function configuredModel(model) {
+	if (model === void 0) throw new TouchpressError({ kind: "ai-not-configured" });
+	return model;
+}
+//#endregion
 //#region src/playwright/fixtures.ts
 const SESSION_FIXTURE_TIMEOUT_MS = 18e4;
 const DEVICE_FIXTURE_TIMEOUT_MS = 12e4;
 /**
-* Tangere's options and none of its fixtures, for a setup project that reads the
+* Touchpress's options and none of its fixtures, for a setup project that reads the
 * configuration before any session exists, such as one calling `preflight`.
 *
 * `platform`, `app`, and `readyWhen` default to `undefined` rather than to a
@@ -35,31 +528,35 @@ const setupTest = test$1.extend({
 		option: true,
 		scope: "worker"
 	}],
-	relaunch: [TANGERE_DEFAULTS.relaunch, {
+	relaunch: [TOUCHPRESS_DEFAULTS.relaunch, {
+		option: true,
+		scope: "worker"
+	}],
+	onDeviceInUse: [TOUCHPRESS_DEFAULTS.onDeviceInUse, {
 		option: true,
 		scope: "worker"
 	}],
-	onDeviceInUse: [TANGERE_DEFAULTS.onDeviceInUse, {
+	settleQuietMs: [TOUCHPRESS_DEFAULTS.settleQuietMs, {
 		option: true,
 		scope: "worker"
 	}],
-	settleQuietMs: [TANGERE_DEFAULTS.settleQuietMs, {
+	launchTimeout: [TOUCHPRESS_DEFAULTS.launchTimeout, {
 		option: true,
 		scope: "worker"
 	}],
-	launchTimeout: [TANGERE_DEFAULTS.launchTimeout, {
+	dismissDevOverlay: [TOUCHPRESS_DEFAULTS.dismissDevOverlay, {
 		option: true,
 		scope: "worker"
 	}],
-	dismissDevOverlay: [TANGERE_DEFAULTS.dismissDevOverlay, {
+	evidence: [TOUCHPRESS_DEFAULTS.evidence, {
 		option: true,
 		scope: "worker"
 	}],
-	evidence: [TANGERE_DEFAULTS.evidence, {
+	sessionPrefix: [TOUCHPRESS_DEFAULTS.sessionPrefix, {
 		option: true,
 		scope: "worker"
 	}],
-	sessionPrefix: [TANGERE_DEFAULTS.sessionPrefix, {
+	aiModel: [void 0, {
 		option: true,
 		scope: "worker"
 	}]
@@ -105,11 +602,11 @@ const test = setupTest.extend({
 		scope: "worker",
 		timeout: SESSION_FIXTURE_TIMEOUT_MS
 	}],
-	device: [async ({ session }, use, testInfo) => {
+	device: [async ({ session, aiModel }, use, testInfo) => {
 		const sink = playwrightSink();
 		if (session.options.relaunch === "per-test" && startedTests.has(session)) await session.relaunch(sink);
 		startedTests.add(session);
-		await use(createDevice(session, sink));
+		await use(withAi(createDevice(session, sink), session, sink, aiModel));
 		if (shouldCapture(testInfo, session.options.evidence)) await captureEvidence(session, sink);
 	}, {
 		auto: true,
@@ -436,4 +933,4 @@ const expect = expect$1.extend({
 	}
 });
 //#endregion
-export { TangereError, expect, preflight, setupTest, test };
+export { TouchpressError, expect, preflight, setupTest, test };