simframe 0.1.0 → 0.4.1

package/src/input.js

@@ -0,0 +1,221 @@
+// Input driver. simframe observes without any of this; input is an optional
+// capability layered on top, so every entry point here has to answer "is this
+// even available?" before it answers anything else.
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const run = promisify(execFile);
+
+const IDB_HINT =
+  'install it with: brew tap facebook/fb && brew install idb-companion && pipx install fb-idb';
+
+let driverCache = null;
+
+/** @returns {Promise<{name: string, available: boolean, version: string|null, reason: string|null}>} */
+export async function detectDriver({ refresh = false } = {}) {
+  if (driverCache && !refresh) return driverCache;
+  try {
+    // idb has no --version; `--help` is the cheapest proof the client runs.
+    await run('idb', ['--help'], { timeout: 8000 });
+    let version = 'installed';
+    try {
+      const { stdout } = await run('idb_companion', ['--version'], { timeout: 5000 });
+      const info = JSON.parse(stdout.trim());
+      version = `companion built ${info.build_date}`;
+    } catch {
+      /* the companion is spawned on demand; its absence surfaces at first use */
+    }
+    driverCache = { name: 'idb', available: true, version, reason: null };
+  } catch (err) {
+    driverCache = {
+      name: 'idb',
+      available: false,
+      version: null,
+      reason:
+        err.code === 'ENOENT'
+          ? `idb is not installed, so simframe can observe the screen but cannot touch it — ${IDB_HINT}`
+          : `idb is present but did not run: ${err.message}`,
+    };
+  }
+  return driverCache;
+}
+
+async function requireDriver() {
+  const driver = await detectDriver();
+  if (!driver.available) throw new Error(driver.reason);
+  return driver;
+}
+
+async function idb(args, { timeout = 20_000 } = {}) {
+  await requireDriver();
+  const { stdout } = await run('idb', args, { timeout, maxBuffer: 32 << 20 });
+  return stdout;
+}
+
+/**
+ * Screen geometry, needed because the accessibility tree speaks in points while
+ * a simframe frame is a scaled bitmap. Without this mapping, a coordinate read
+ * off an image lands in the wrong place.
+ */
+const geometryCache = new Map();
+
+/** Cached: geometry costs an idb round trip and never changes while booted. */
+export async function screenInfo(udid, { refresh = false } = {}) {
+  if (!refresh && geometryCache.has(udid)) return geometryCache.get(udid);
+  const info = await readScreenInfo(udid);
+  geometryCache.set(udid, info);
+  return info;
+}
+
+async function readScreenInfo(udid) {
+  const out = await idb(['describe', '--json', '--udid', udid]);
+  const info = JSON.parse(out.trim().split('\n').filter(Boolean).pop());
+  const dims = info.screen_dimensions || {};
+  const density = dims.density || 1;
+  return {
+    pixelWidth: dims.width ?? null,
+    pixelHeight: dims.height ?? null,
+    density,
+    pointWidth: dims.width_points ?? (dims.width ? Math.round(dims.width / density) : null),
+    pointHeight: dims.height_points ?? (dims.height ? Math.round(dims.height / density) : null),
+  };
+}
+
+/** The accessibility tree, flattened. This is what makes tap-by-label possible. */
+export async function describeAll(udid) {
+  // Passing --json here yields empty output; the default already emits JSON.
+  const out = await idb(['ui', 'describe-all', '--udid', udid]);
+  const nodes = [];
+  for (const line of out.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const parsed = JSON.parse(trimmed);
+      const stack = Array.isArray(parsed) ? [...parsed] : [parsed];
+      while (stack.length) {
+        const node = stack.shift();
+        nodes.push(normalizeNode(node));
+        if (Array.isArray(node.children)) stack.unshift(...node.children);
+      }
+    } catch {
+      /* idb interleaves non-JSON status lines; skip them */
+    }
+  }
+  return nodes.filter((n) => n.frame);
+}
+
+// Icon fonts put glyphs in the Unicode private use areas, so a label arrives as
+// "<glyph>, My Tools". Matching has to see through that to the readable text.
+const PRIVATE_USE = /[\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}]/gu;
+
+export function cleanLabel(label) {
+  if (!label) return label ?? null;
+  const stripped = label.replace(PRIVATE_USE, '');
+  return stripped.replace(/\s*,\s*/g, ', ').replace(/^[,\s]+|[,\s]+$/g, '').trim() || null;
+}
+
+function normalizeNode(node) {
+  const frame = node.frame || node.AXFrame || null;
+  const rawLabel = node.AXLabel ?? node.label ?? null;
+  return {
+    label: cleanLabel(rawLabel),
+    rawLabel,
+    value: node.AXValue ?? node.value ?? null,
+    type: node.type ?? node.AXType ?? null,
+    identifier: node.AXUniqueId ?? node.identifier ?? null,
+    enabled: node.AXEnabled ?? node.enabled ?? null,
+    frame: frame
+      ? {
+          x: frame.x ?? frame.X ?? 0,
+          y: frame.y ?? frame.Y ?? 0,
+          width: frame.width ?? frame.Width ?? 0,
+          height: frame.height ?? frame.Height ?? 0,
+        }
+      : null,
+    raw: node,
+  };
+}
+
+const text = (n) => [n.label, n.value, n.identifier].filter(Boolean).join(' ');
+
+/**
+ * Find the element a caller means. Exact label first, then identifier, then a
+ * case-insensitive substring — and an ambiguous match is an error rather than a
+ * guess, because a wrong tap is worse than no tap.
+ */
+export function matchElement(nodes, query, { index } = {}) {
+  const q = String(query).toLowerCase();
+  const tiers = [
+    nodes.filter((n) => (n.label ?? '').toLowerCase() === q),
+    nodes.filter((n) => (n.identifier ?? '').toLowerCase() === q),
+    nodes.filter((n) => text(n).toLowerCase().includes(q)),
+  ];
+  for (const tier of tiers) {
+    if (!tier.length) continue;
+    if (index != null) {
+      if (index >= tier.length) {
+        throw new Error(`"${query}" matched ${tier.length} elements; index ${index} is out of range`);
+      }
+      return tier[index];
+    }
+    if (tier.length > 1) {
+      const shown = tier.slice(0, 6).map((n, i) => `[${i}] ${text(n) || n.type}`).join(', ');
+      throw new Error(
+        `"${query}" matched ${tier.length} elements — pass index to choose: ${shown}`,
+      );
+    }
+    return tier[0];
+  }
+  throw new Error(`no element matching "${query}" is on screen`);
+}
+
+export function centerOf(node) {
+  return {
+    x: Math.round(node.frame.x + node.frame.width / 2),
+    y: Math.round(node.frame.y + node.frame.height / 2),
+  };
+}
+
+export async function tapPoint(udid, x, y, { durationMs } = {}) {
+  const args = ['ui', 'tap', '--udid', udid, String(Math.round(x)), String(Math.round(y))];
+  if (durationMs) args.push('--duration', String(durationMs / 1000));
+  await idb(args);
+  return { x: Math.round(x), y: Math.round(y) };
+}
+
+export async function tapLabel(udid, query, { index, durationMs } = {}) {
+  const node = matchElement(await describeAll(udid), query, { index });
+  const point = centerOf(node);
+  await tapPoint(udid, point.x, point.y, { durationMs });
+  return { node, point };
+}
+
+export async function typeText(udid, value) {
+  await idb(['ui', 'text', '--udid', udid, String(value)]);
+}
+
+export async function pressKey(udid, keycode) {
+  await idb(['ui', 'key', '--udid', udid, String(keycode)]);
+}
+
+export async function pressButton(udid, name) {
+  await idb(['ui', 'button', '--udid', udid, String(name).toUpperCase()]);
+}
+
+export async function swipe(udid, from, to, { durationMs = 300 } = {}) {
+  await idb([
+    'ui', 'swipe', '--udid', udid,
+    String(Math.round(from.x)), String(Math.round(from.y)),
+    String(Math.round(to.x)), String(Math.round(to.y)),
+    '--duration', String(durationMs / 1000),
+  ]);
+}
+
+/** Map a coordinate read off a simframe image into the points idb expects. */
+export function imageToPoints({ x, y }, { imageWidth, imageHeight, pointWidth, pointHeight }) {
+  if (!pointWidth || !pointHeight) throw new Error('screen geometry is unknown');
+  return {
+    x: Math.round((x / imageWidth) * pointWidth),
+    y: Math.round((y / imageHeight) * pointHeight),
+  };
+}

package/src/intent.js

@@ -0,0 +1,100 @@
+// Human-level steps.
+//
+// A person filling an unfamiliar form does not study it. They pick something in
+// the picker and hit whatever confirming button is on screen. Reproducing that
+// instinct removes the expensive part of driving a UI with an agent: the model
+// round trip spent reasoning about controls it has never seen.
+import * as input from './input.js';
+
+/** Confirming words, most specific first. The first tier with a hit wins. */
+const CONFIRM_TIERS = [
+  [/^apply$/i, /^confirm$/i, /^ok$/i],
+  [/^done$/i, /^submit$/i, /^save$/i],
+  [/^continue$/i, /^next$/i, /^yes$/i],
+  [/^select$/i, /^choose$/i, /^add$/i],
+];
+
+const DISMISS = /^(cancel|reset|close|back|no|clear|dismiss)$/i;
+
+export function onScreen(node, geo) {
+  const f = node.frame;
+  return f && f.y >= 0 && f.y < (geo?.pointHeight ?? 874) && f.x >= 0 && f.x < (geo?.pointWidth ?? 402);
+}
+
+/**
+ * The control a person would press to commit what is on screen. Prefers enabled
+ * controls, then the lowest one, then the rightmost — confirm sits bottom-right
+ * of cancel by near-universal convention.
+ */
+export function findConfirm(nodes, geo) {
+  const usable = nodes.filter(
+    (n) => n.label && onScreen(n, geo) && !DISMISS.test(n.label.trim()),
+  );
+  for (const tier of CONFIRM_TIERS) {
+    const hits = usable.filter((n) => tier.some((re) => re.test(n.label.trim())));
+    if (!hits.length) continue;
+    return hits.sort(
+      (a, b) =>
+        (b.enabled !== false) - (a.enabled !== false) ||
+        b.frame.y - a.frame.y ||
+        b.frame.x - a.frame.x,
+    )[0];
+  }
+  return null;
+}
+
+/**
+ * Selectable options in an open picker: labelled rows that are neither the
+ * confirm/dismiss row nor the search box, sharing a left edge and a row height.
+ */
+export function findOptions(nodes, geo) {
+  const confirm = findConfirm(nodes, geo);
+  const floor = confirm ? confirm.frame.y - 8 : (geo?.pointHeight ?? 874);
+  const rows = nodes.filter(
+    (n) =>
+      n.label &&
+      onScreen(n, geo) &&
+      n.frame.y > 120 &&
+      n.frame.y < floor &&
+      n.frame.height >= 12 &&
+      n.frame.height <= 70 &&
+      !DISMISS.test(n.label.trim()) &&
+      !CONFIRM_TIERS.flat().some((re) => re.test(n.label.trim())) &&
+      !/^search$/i.test(n.label.trim()) &&
+      !/required|please select/i.test(n.label),
+  );
+  // Picker rows repeat a left edge; that is what separates them from headings.
+  const byX = new Map();
+  for (const r of rows) {
+    const key = Math.round(r.frame.x / 8) * 8;
+    byX.set(key, [...(byX.get(key) || []), r]);
+  }
+  const biggest = [...byX.values()].sort((a, b) => b.length - a.length)[0] || [];
+  return biggest.length >= 2 ? biggest.sort((a, b) => a.frame.y - b.frame.y) : rows;
+}
+
+/** Pick an option without caring which — the human move in an unfamiliar picker. */
+export async function chooseAny(udid, { prefer, geo } = {}) {
+  const nodes = await input.describeAll(udid);
+  const options = findOptions(nodes, geo);
+  if (!options.length) throw new Error('no selectable options found on screen');
+  const chosen =
+    (prefer && options.find((o) => o.label.toLowerCase().includes(String(prefer).toLowerCase()))) ||
+    options[0];
+  const point = input.centerOf(chosen);
+  await input.tapPoint(udid, point.x, point.y);
+  return { label: chosen.label, point, optionCount: options.length };
+}
+
+export async function confirm(udid, { geo } = {}) {
+  const nodes = await input.describeAll(udid);
+  const target = findConfirm(nodes, geo);
+  if (!target) throw new Error('no confirming control on screen');
+  const point = input.centerOf(target);
+  await input.tapPoint(udid, point.x, point.y);
+  return { label: target.label, point, enabled: target.enabled };
+}
+
+export function isTextInput(node) {
+  return /TextField|TextView|SearchField/i.test(node.type || '');
+}