simframe 0.6.1 → 0.7.0

package/src/index.js

@@ -19,7 +19,7 @@ import * as graph from './graph.js';
 import * as matching from './matching.js';
 import * as refs from './refs.js';
 import * as screenmap from './screenmap.js';
-import { resolveDevice, resize, screenshot } from './simctl.js';
+import { capabilitiesFor, resolveDevice, resize, screenshot } from './platform/index.js';
 import * as store from './store.js';
 
 const HERE = path.dirname(fileURLToPath(import.meta.url));
@@ -186,15 +186,24 @@ export function fallbackReason(udid) {
 }
 
 /**
- * Start whichever engine was asked for.
+ * Start whichever engine was asked for, out of the ones this device's platform
+ * has.
  *
- * simframed unless told otherwise: it reads the framebuffer directly and is
- * roughly thirty times faster per frame. The simctl loop stays reachable with
- * `engine: 'simctl'`, and is used automatically when the daemon cannot be
- * built — a machine with no Swift toolchain still has to work.
+ * On iOS that is simframed unless told otherwise: it reads the framebuffer
+ * directly and is roughly thirty times faster per frame, and the screenshot
+ * loop stays reachable with `engine: 'screenshot'` for a machine with no Swift
+ * toolchain. On Android the loop is the only engine there is — and asking for
+ * simframed there is refused rather than attempted, because a Swift daemon
+ * built against CoreSimulator has nothing to say to an emulator, and the
+ * failure it produces says nothing useful about why.
  */
 async function startEngine(udid, options) {
-  if ((options.engine ?? 'simframed') === 'simframed') {
+  const supported = capabilitiesFor(udid).captureEngines;
+  const wanted = engine.normalizeEngine(options.engine ?? supported[0]);
+  if (!supported.includes(wanted)) {
+    throw new Error(`this device cannot run the ${wanted} capture engine — it supports ${supported.join(', ')}`);
+  }
+  if (wanted === 'simframed') {
     const built = await engine.ensureBuilt();
     if (built.ok) {
       engineFallbackReason = null;
@@ -206,7 +215,7 @@ async function startEngine(udid, options) {
     recordFallback(udid, engineFallbackReason);
   }
   spawnNodeDaemon(udid, options);
-  return 'simctl';
+  return 'screenshot';
 }
 
 function spawnNodeDaemon(udid, options) {
@@ -289,12 +298,38 @@ export function changeLevel(diff) {
   return 'none';
 }
 
+/** How long a stall has been going on, in words an agent can act on. */
+function stallNote(health) {
+  const forMs = Math.max(0, Date.now() - (health.since ?? Date.now()));
+  const parts = [`capture: stalled — the display surface has been unreadable for ${Math.round(forMs / 1000)}s`];
+  if (health.reattaches) parts.push(`${health.reattaches} re-attach${health.reattaches === 1 ? '' : 'es'} did not help`);
+  if (health.reason) parts.push(String(health.reason).slice(0, 120));
+  // The cure is the user's to apply. Saying so is the difference between an
+  // agent that reports "the simulator is wedged" and one that retries a tap
+  // twenty times because nothing appeared to change.
+  parts.push('only restarting the device is known to cure it');
+  return parts.join('; ');
+}
+
 export function liveness(udid, state) {
   const ageMs = Date.now() - state.capturedAt;
   const { running } = daemonStatus(udid);
+  // A wedged device and a quiet one look identical from the frames alone: both
+  // produce nothing. The difference is that a wedged one is failing reads, and
+  // only the capture loop knows that, so it writes it down.
+  const health = store.captureHealth(udid);
+  const stalled = Boolean(health?.stalled);
   if (!running) {
-    return { ok: false, ageMs, note: 'the capture loop has died; the frame you are looking at is the last one it wrote' };
+    return {
+      ok: false,
+      ageMs,
+      stalled,
+      note: stalled
+        ? `the capture loop has died, and it was stalled before it did — ${stallNote(health)}`
+        : 'the capture loop has died; the frame you are looking at is the last one it wrote',
+    };
   }
+  if (stalled) return { ok: false, ageMs, stalled: true, note: stallNote(health) };
   // Frame age means "stalled" only for a fixed-rate loop.
   //
   // simframed captures on damage, so a screen that is genuinely still produces
@@ -306,9 +341,9 @@ export function liveness(udid, state) {
   // daemon.
   const damageDriven = engine.runningEngine(udid) === 'simframed';
   if (!damageDriven && ageMs > STALE_FRAME_MS) {
-    return { ok: false, ageMs, note: `capture loop is stalled: newest frame is ${ageMs}ms old` };
+    return { ok: false, ageMs, stalled: false, note: `capture loop is stalled: newest frame is ${ageMs}ms old` };
   }
-  return { ok: true, ageMs, note: null };
+  return { ok: true, ageMs, stalled: false, note: null };
 }
 
 /**
@@ -502,7 +537,16 @@ export async function waitFor(
       // a button changing state. Waiting the full timeout for a change that
       // will never be visible turns a 100ms action into a 12s one, so give up
       // early and say so, rather than silently burning the clock.
-      if (!sawChange && Date.now() - startedAt > reactionMs && state.stableForMs >= stableMs) {
+      //
+      // But "nothing changed" is a claim about something observed, and with no
+      // frame captured since this call began, nothing has been. The screenshot
+      // engine idles at 1.5 fps, so a 500 ms reaction window expired before the
+      // first new frame existed: a tap that opened a whole activity was
+      // reported as having no visible effect, and the text meant for the field
+      // it opened was typed into nothing. Damage-driven capture on iOS hid this
+      // by being fast.
+      const observedSomething = state.seq - startSeq >= 1;
+      if (!sawChange && observedSomething && Date.now() - startedAt > reactionMs && state.stableForMs >= stableMs) {
         return done(false, { noVisibleChange: true });
       }
 
@@ -724,6 +768,31 @@ export async function settledState(udid, { settleMs = MEMORY_SETTLE_MS, timeoutM
   return { state, settled: false };
 }
 
+/**
+ * Read the screen with the perception layers pinned, and keep nothing.
+ *
+ * Every other path decides for itself which layers to read, which is right for
+ * doing work and useless for measuring: the question "what is the accessibility
+ * tier worth" needs the same frame read twice, once with it and once without.
+ * `persist: false` so measuring teaches the graph nothing.
+ */
+export async function readScreenWith(deviceQuery, { useAx = true, useOcr = true, options } = {}) {
+  const { device, state } = await ensureDaemon(deviceQuery, options);
+  const udid = device.udid;
+  const geo = await deviceGeometry(udid, state);
+  const entry = await screenmap.build(udid, {
+    hash: state.hash,
+    layoutHash: state.layoutHash,
+    fullFrame: await fullFrameFor(udid, state),
+    density: geo.density,
+    screen: { width: geo.pointWidth, height: geo.pointHeight },
+    useAx,
+    useOcr,
+    persist: false,
+  });
+  return { device, entry, points: { width: geo.pointWidth, height: geo.pointHeight } };
+}
+
 export async function locate(
   deviceQuery,
   query,

package/src/input.js

@@ -3,6 +3,7 @@
 // even available?" before it answers anything else.
 import { execFile } from 'node:child_process';
 import * as control from './control.js';
+import { geometryFor, inputDriverFor } from './platform/index.js';
 import { promisify } from 'node:util';
 
 const run = promisify(execFile);
@@ -20,6 +21,12 @@ let driverCache = null;
  * iOS. idb remains the fallback so a machine without the daemon still works.
  */
 export async function driverFor(udid) {
+  // A platform that carries its own input path answers first: there is no
+  // daemon to ask and no idb to fall back to, and reporting either for an
+  // Android emulator is how doctor came to claim "input driver: idb" about a
+  // tool that has never spoken to one.
+  const own = udid ? inputDriverFor(udid) : null;
+  if (own) return { name: own.id, available: true, version: own.detail, reason: null, viaSocket: false };
   if (udid && control.available(udid)) {
     try {
       const status = await control.status(udid);
@@ -140,6 +147,15 @@ export async function screenInfo(udid, { refresh = false } = {}) {
 }
 
 async function readScreenInfo(udid) {
+  // The platform first, where it can answer at all: on Android it is the only
+  // source, and the alternative is `deviceGeometry`'s last-resort guess, which
+  // is an iPhone's numbers and silently wrong for everything else.
+  try {
+    const geo = await geometryFor(udid);
+    if (geo?.pointWidth && geo?.pointHeight) return geo;
+  } catch {
+    /* the backend could not say; the daemon or idb may still be able to */
+  }
   // Ask the daemon first. It holds the device's own point size and scale, which
   // makes it both authoritative and free — and it means geometry no longer
   // needs idb at all. Going to idb first meant a machine without idb could
@@ -300,6 +316,11 @@ export function centerOf(node) {
 
 export async function tapPoint(udid, x, y, { durationMs } = {}) {
   const point = { x: Math.round(x), y: Math.round(y) };
+  const own = inputDriverFor(udid);
+  if (own) {
+    await own.tap(udid, point.x, point.y, durationMs ? { durationMs } : {});
+    return point;
+  }
   if (control.available(udid)) {
     await control.tap(udid, point.x, point.y, durationMs ? { durationMs } : {});
     return point;
@@ -318,6 +339,15 @@ export async function tapLabel(udid, query, { index, durationMs } = {}) {
 }
 
 export async function typeText(udid, value) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    // No pasteboard on Android (docs/DEFERRED.md), so exact text goes through
+    // the same keystroke path as everything else. `event text` carries
+    // characters rather than key positions, so a non-Latin host layout does not
+    // reinterpret them — which is the reason the pasteboard exists on iOS.
+    await own.text(udid, String(value));
+    return;
+  }
   if (control.available(udid)) {
     // The daemon's paste path carries characters rather than key positions, so
     // it is not reinterpreted by the device's keyboard layout.
@@ -329,6 +359,11 @@ export async function typeText(udid, value) {
 
 /** Key events rather than text: for shortcuts and search-as-you-type. */
 export async function typeKeys(udid, value) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    await own.text(udid, String(value));
+    return;
+  }
   if (control.available(udid)) {
     await control.type(udid, String(value));
     return;
@@ -337,6 +372,11 @@ export async function typeKeys(udid, value) {
 }
 
 export async function pressKey(udid, keycode) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    await own.key(udid, keycode);
+    return;
+  }
   await idb(['ui', 'key', '--udid', udid, String(keycode)]);
 }
 
@@ -363,6 +403,13 @@ export async function resetSession(udid) {
 }
 
 export async function pressButton(udid, name) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    // Android's whole key vocabulary is safe to offer: `input keyevent` takes
+    // names through a public API, so unlike Indigo there is nothing to guess.
+    await own.key(udid, name);
+    return;
+  }
   if (control.available(udid)) {
     try {
       await control.press(udid, String(name).toLowerCase());
@@ -376,6 +423,11 @@ export async function pressButton(udid, name) {
 }
 
 export async function swipe(udid, from, to, { durationMs = 300 } = {}) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    await own.swipe(udid, from, to, { durationMs });
+    return;
+  }
   if (control.available(udid)) {
     await control.swipe(udid, from, to, { durationMs });
     return;

package/src/mcp.js

@@ -14,14 +14,14 @@ import * as actions from './actions.js';
 import * as api from './index.js';
 import * as input from './input.js';
 import * as navigate from './navigate.js';
-import { bootedDevices, PERMISSION_SERVICES } from './simctl.js';
+import { bootedDevices, permissionServices } from './platform/index.js';
 import * as store from './store.js';
 import * as view from './view.js';
 
 const deviceProp = {
   device: {
     type: 'string',
-    description: 'Simulator UDID or name substring. Defaults to the booted simulator.',
+    description: 'Device UDID or name substring — a simulator udid or an emulator serial. Defaults to the booted device.',
   },
 };
 
@@ -180,7 +180,7 @@ const TOOLS = [
   },
   {
     name: 'sim_open_url',
-    description: 'Open a URL or deep link on the simulator — the fastest way to reach a screen when the app has a link for it.',
+    description: 'Open a URL or deep link on the device — the fastest way to reach a screen when the app has a link for it.',
     inputSchema: {
       type: 'object',
       properties: { ...deviceProp, url: { type: 'string' } },
@@ -189,7 +189,7 @@ const TOOLS = [
   },
   {
     name: 'sim_permission',
-    description: `Grant, revoke or reset a privacy permission for an app. Do this instead of tapping the system alert: the alert is not part of the app under test, and its buttons move between iOS versions. Services: ${PERMISSION_SERVICES.join(', ')}.`,
+    description: `Grant, revoke or reset a privacy permission for an app. Do this instead of tapping the system alert: the alert is not part of the app under test, and its buttons move between OS versions. Not every service exists on every platform — the device's own backend refuses one it does not have. Services: ${permissionServices().join(', ')}.`,
     inputSchema: {
       type: 'object',
       properties: {
@@ -304,7 +304,7 @@ const TOOLS = [
   },
   {
     name: 'sim_devices',
-    description: 'List booted iOS simulators that simframe can capture.',
+    description: 'List the booted devices simframe can drive — iOS simulators and Android emulators.',
     inputSchema: { type: 'object', properties: {} },
   },
 ];
@@ -826,7 +826,7 @@ function listStateDirs() {
 
 async function devices() {
   const booted = await bootedDevices();
-  if (!booted.length) return { content: [text('no booted simulators')] };
+  if (!booted.length) return { content: [text('no booted devices')] };
   return {
     content: [text(booted.map((d) => `${d.name} · ${d.runtime} · ${d.udid}`).join('\n'))],
   };