simframe 0.6.2 → 0.7.0

package/src/cli.js

@@ -2,7 +2,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { runDaemon, DEFAULTS } from './daemon.js';
-import { bootedDevices, listDevices, resolveDevice } from './simctl.js';
+import { bootedDevices, capabilitiesFor, listDevices, PLATFORMS, resolveDevice, toolchainChecks } from './platform/index.js';
 import * as actions from './actions.js';
 import * as api from './index.js';
 import * as input from './input.js';
@@ -51,8 +51,8 @@ Options
   --json                 machine-readable output — on every command
   --out=<file>           output path for frame/strip/recall
   --detail=low|normal|high|full   or --detail=<max pixels>
-  --engine=simframed|simctl   capture engine (default simframed)
-  --fps=<n>              capture rate while the screen is moving (simctl engine only)
+  --engine=simframed|screenshot  capture engine (default: the fastest the device has)
+  --fps=<n>              capture rate while the screen is moving (screenshot engine only)
   --count=<n>            frames in a strip (default 5)
   --since=<hash|seq>     compare against this frame (see: simframe mark)
   --mode=settle|change|stable   what wait waits for (default settle)
@@ -177,18 +177,21 @@ async function main() {
     case 'start': {
       const { device: dev, state, started } = await api.ensureDaemon(device, options);
       const engineModule = await import('./engine.js');
-      const running = engineModule.runningEngine(dev.udid) ?? 'simctl';
+      const best = capabilitiesFor(dev.udid).captureEngines[0];
+      const running = engineModule.runningEngine(dev.udid) ?? best;
       console.log(
         `${started ? 'started' : 'already running'} — ${dev.name} (${dev.runtime}) ` +
           `engine=${running} frame #${state.seq} ${state.width}x${state.height}`,
       );
       // Say which engine, and if it is the slow one, say why. A downgrade that
-      // prints nothing is how this shipped broken twice.
-      if (running !== 'simframed') {
+      // prints nothing is how this shipped broken twice. But it is only a
+      // downgrade if this platform has something better: the screenshot loop is
+      // the whole of Android's capture, not a fallback from anything.
+      if (running !== best) {
         const why = api.fallbackReason(dev.udid);
         console.log(
-          `WARN engine=simctl — roughly 30x slower per frame. ` +
-            (why ? `simframed unavailable: ${why}` : 'reason unrecorded; run simframe doctor'),
+          `WARN engine=${running} — roughly 30x slower per frame than ${best}. ` +
+            (why ? `${best} unavailable: ${why}` : 'reason unrecorded; run simframe doctor'),
         );
         if (Boolean(flags.strict) || process.env.SIMFRAME_STRICT === '1') {
           console.error('--strict: refusing to run on a degraded engine');
@@ -697,7 +700,7 @@ async function main() {
       if (flags.json) {
         console.log(JSON.stringify(shown, null, 2));
       } else if (!shown.length) {
-        console.log('no booted simulators (pass --all to list every device)');
+        console.log('no booted devices (pass --all to list every device)');
       } else {
         for (const d of shown) console.log(`${d.state === 'Booted' ? '●' : '○'} ${d.name}  ${d.runtime}  ${d.udid}`);
       }
@@ -748,11 +751,10 @@ async function doctor({ json = false, strict = false, device } = {}) {
 
   add('node', 'ok', process.version);
   const { execFileSync } = await import('node:child_process');
-  try {
-    add('xcrun', 'ok', execFileSync('xcrun', ['--version'], { encoding: 'utf8' }).trim().split('\n')[0]);
-  } catch (err) {
-    add('xcrun', 'fail', err.message);
-  }
+  // The active backend names its own prerequisites — doctor renders them and
+  // does not know what they are. On iOS that is xcrun; on Android it will be
+  // adb, and this line will not change.
+  for (const check of toolchainChecks()) add(check.name, check.level, check.detail);
   try {
     execFileSync('sips', ['--version'], { encoding: 'utf8', stdio: 'pipe' });
     add('sips', 'ok', 'available');
@@ -798,11 +800,21 @@ async function doctor({ json = false, strict = false, device } = {}) {
       const wanted = await resolveDevice(device);
       booted = booted.filter((d) => d.udid === wanted.udid);
     }
-    add('booted simulator', booted.length ? 'ok' : 'warn',
+    // `deviceNoun` earns its place here: one platform's devices are called by
+    // its own word, and a mixed set by the neutral one. An emulator reported as
+    // a "booted simulator" is the same small lie as an emulator reported as
+    // having an idb input driver.
+    const nouns = [...new Set(booted.map((d) => capabilitiesFor(d.udid) && PLATFORMS[d.platform].deviceNoun))];
+    add(`booted ${nouns.length === 1 ? nouns[0] : 'device'}`, booted.length ? 'ok' : 'warn',
       booted.map((d) => `${d.name} (${d.runtime})`).join(', ') || 'none');
     for (const d of booted) {
       const input = await import('./input.js');
       const control = await import('./control.js');
+      // What this device's platform can do at all. Without asking, doctor
+      // described an Android emulator in iOS terms — "input driver: idb" about
+      // a tool that has never spoken to one.
+      const caps = capabilitiesFor(d.udid);
+      const bestEngine = caps.captureEngines[0];
       // Start the engine before asking which engine is in use. Reading it first
       // reports `simctl` on any machine where nothing happens to be running
       // yet — a warning about a downgrade that has not occurred, and one that
@@ -813,27 +825,43 @@ async function doctor({ json = false, strict = false, device } = {}) {
       // ensureDaemon waits for a frame; the control socket comes up a moment
       // later. Asking immediately reports `idb` for a device whose own input
       // path is seconds from ready — a race that would read as CI flake.
-      for (let i = 0; i < 40 && !control.available(d.udid); i += 1) {
+      for (let i = 0; i < 40 && caps.input.supported && !control.available(d.udid); i += 1) {
         await new Promise((r) => setTimeout(r, 50));
       }
-      const driver = await input.driverFor(d.udid, { refresh: true });
+      const driver = caps.input.supported ? await input.driverFor(d.udid, { refresh: true }) : null;
       // Which engine is actually capturing, from the daemon's own record.
       // `control.available` answers a different question — whether the input
       // socket is up — and using it here reported simctl on a machine that was
       // capturing with simframed perfectly well.
-      const captureEngine = engineModule.runningEngine(d.udid) ?? 'simctl';
+      const captureEngine = engineModule.runningEngine(d.udid) ?? bestEngine;
       const daemon = captureEngine === 'simframed';
-      const why = captureEngine === 'simctl' ? api.fallbackReason(d.udid) : null;
-      add(`capture engine (${d.name})`, captureEngine === 'simframed' ? 'ok' : 'warn',
-        captureEngine === 'simframed'
-          ? 'simframed'
-          : `simctl — roughly 30x slower per frame${why ? `; simframed unavailable: ${why}` : '. Run simframe start to see why'}`,
+      const why = captureEngine === bestEngine ? null : api.fallbackReason(d.udid);
+      add(`capture engine (${d.name})`, captureEngine === bestEngine ? 'ok' : 'warn',
+        captureEngine === bestEngine
+          ? captureEngine
+          : `${captureEngine} — roughly 30x slower per frame than ${bestEngine}${why ? `; ${bestEngine} unavailable: ${why}` : '. Run simframe start to see why'}`,
         { key: 'capture.engine', value: captureEngine });
-      add(`input driver (${d.name})`, driver.available ? (driver.name === 'simframed' ? 'ok' : 'warn') : 'warn',
-        driver.available ? `${driver.name}: ${driver.version}` : driver.reason,
-        { key: 'input.driver', value: driver.available ? driver.name : null });
+      // A layer this platform does not have yet is `optional`, the level that
+      // means "documented as absent" rather than "this machine is degraded".
+      if (!caps.input.supported) {
+        add(`input driver (${d.name})`, 'optional', caps.input.note, { key: 'input.driver', value: null });
+      } else {
+        // `warn` means this machine could be doing better and silently is not —
+        // which is true of idb on a simulator and false of the platform's own
+        // driver. The console is not a downgrade on Android; it is the only
+        // input path there is, and grading it a downgrade made `--strict` fail
+        // on a device that was working perfectly.
+        const best = driver.name === 'simframed' || driver.name === caps.input.via;
+        add(`input driver (${d.name})`, driver.available ? (best ? 'ok' : 'warn') : 'warn',
+          driver.available ? `${driver.name}: ${driver.version}` : driver.reason,
+          { key: 'input.driver', value: driver.available ? driver.name : null });
+      }
       add(`text recognition (${d.name})`, 'ok',
         daemon ? 'simframed (in-process, off the framebuffer)' : 'sips + helper binary');
+      if (!caps.ax.supported) {
+        add(`accessibility tree (${d.name})`, 'optional', caps.ax.note, { key: 'ax.driver', value: null });
+        continue;
+      }
       const ax = await input.axDriverFor(d.udid);
       // idb here is a downgrade unless it was asked for. `warn` means this
       // machine could be doing better and silently is not; a driver someone
@@ -849,6 +877,14 @@ async function doctor({ json = false, strict = false, device } = {}) {
       add('capture', 'ok',
         `frame #${res.state.seq} ${res.width}x${res.height} in ${Date.now() - t0}ms (age ${res.ageMs}ms)`,
         { key: 'capture.frames', value: res.state.seq });
+      // A wedged device produces the same nothing as a quiet one, so doctor has
+      // to ask the capture loop rather than look at the frames. `fail`, not
+      // `warn`: nothing here is degraded-but-working, and the cure is a device
+      // restart that simframe deliberately does not perform.
+      for (const d of booted) {
+        const live = api.liveness(d.udid, (await api.getState(d.udid)).state);
+        if (live.stalled) add(`capture health (${d.name})`, 'fail', live.note, { key: 'capture.stalled', value: true });
+      }
     }
   } catch (err) {
     add('capture', 'fail', err.message);

package/src/daemon.js

@@ -12,7 +12,7 @@ import {
   signatureToHex,
 } from './analyze.js';
 import * as store from './store.js';
-import { isBootedSync, resize, screenshot } from './simctl.js';
+import { isBootedSync, resize, screenshot } from './platform/index.js';
 
 // Bump whenever the shape of state.json changes, so an upgraded client retires
 // a capture loop left running by an older install instead of misreading it.
@@ -25,6 +25,18 @@ import { isBootedSync, resize, screenshot } from './simctl.js';
 // through a flow. A unit test asserts these two constants match.
 export const STATE_VERSION = 6;
 
+/**
+ * How many failed captures in a row mean this loop is wedged rather than
+ * unlucky.
+ *
+ * Four, against the ten that make it give up: far enough in that a single
+ * hiccup does not raise an alarm, early enough that a reader learns about it
+ * while the loop is still trying. The Swift daemon reaches the same conclusion
+ * differently — it counts re-resolves of the display port, because there a
+ * successful re-resolve resets the failure count and hides the loop.
+ */
+export const STALLED_AFTER_ERRORS = 4;
+
 export const DEFAULTS = {
   fps: 4,
   idleFps: 1.5,
@@ -85,6 +97,7 @@ export async function runDaemon(device, options = {}) {
   let ringIndex = [];
   let lastChangeAt = Date.now();
   let consecutiveErrors = 0;
+  let stalledSince = null;
   let lastBootCheck = Date.now();
   let running = true;
   const stop = () => {
@@ -141,6 +154,10 @@ export async function runDaemon(device, options = {}) {
 
       seq = nextSeq;
       prevSignature = signature;
+      if (consecutiveErrors >= STALLED_AFTER_ERRORS) {
+        log('capture recovered on its own');
+        store.writeCaptureHealth(udid, null);
+      }
       consecutiveErrors = 0;
 
       const hash = frameHash(bmp);
@@ -199,7 +216,25 @@ export async function runDaemon(device, options = {}) {
     } catch (err) {
       consecutiveErrors++;
       log(`capture error (${consecutiveErrors}): ${err.message}`);
+      // Say that capture is wedged rather than merely slow, and do nothing
+      // about it: the cure is a device restart, and that is the user's to make.
+      // Published rather than only logged, because a reader of `state` sees the
+      // last healthy frame with nothing in it to say the device stopped
+      // answering — the same frames a merely idle screen produces.
+      if (consecutiveErrors >= STALLED_AFTER_ERRORS) {
+        stalledSince ??= Date.now();
+        store.writeCaptureHealth(udid, {
+          stalled: true,
+          since: stalledSince,
+          at: Date.now(),
+          consecutiveFailures: consecutiveErrors,
+          reattaches: 0,
+          reason: err.message,
+        });
+      }
       if (consecutiveErrors >= 10) {
+        // Left published on purpose. The file is how a reader learns why this
+        // loop is not running any more.
         log('exit: too many consecutive capture errors');
         break;
       }

package/src/engine.js

@@ -1,10 +1,16 @@
 // Chooses and starts the capture engine.
 //
 // Two exist: `simframed`, a Swift daemon that reads the framebuffer directly,
-// and `simctl`, the original loop that shells out for each screenshot. The
-// daemon is the default because it is roughly thirty times faster, but the old
-// loop stays reachable — a machine without a Swift toolchain, or an Xcode
-// version where a private symbol has moved, still needs to work.
+// and `screenshot`, the loop that asks the platform boundary for one frame at a
+// time. The daemon is the default on iOS because it is roughly thirty times
+// faster, but the loop stays reachable — a machine without a Swift toolchain,
+// or an Xcode version where a private symbol has moved, still needs to work.
+//
+// The loop used to be called `simctl`, after the tool it shelled out to. It no
+// longer shells out to anything in particular: on Android the same loop reaches
+// the emulator console and captures a frame in ~41 ms, which is not `simctl` by
+// any reading. `simctl` stays accepted as an alias, because it is in shipped
+// meta.json files, in documentation and in people's shell history.
 import { execFile, spawn } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
@@ -17,7 +23,14 @@ const HERE = path.dirname(fileURLToPath(import.meta.url));
 const PACKAGE = path.join(HERE, '..', 'native', 'simframed');
 const BINARY = path.join(PACKAGE, '.build', 'release', 'simframed');
 
-export const ENGINES = ['simframed', 'simctl'];
+export const ENGINES = ['simframed', 'screenshot'];
+
+/** `simctl` was this engine's name until it ran on a second platform. */
+const ENGINE_ALIASES = { simctl: 'screenshot' };
+
+export function normalizeEngine(name) {
+  return ENGINE_ALIASES[name] ?? name;
+}
 
 export function binaryPath() {
   return BINARY;
@@ -95,5 +108,5 @@ export function spawnDaemon(udid, { maxDim, minIntervalMs, idleExitMs } = {}) {
 export function runningEngine(udid) {
   const meta = store.readJson(path.join(store.deviceDir(udid), 'meta.json'));
   if (!meta || !store.isProcessAlive(meta.pid)) return null;
-  return meta.options?.engine === 'simframed' ? 'simframed' : 'simctl';
+  return meta.options?.engine === 'simframed' ? 'simframed' : 'screenshot';
 }

package/src/fingerprint.js

@@ -20,8 +20,12 @@ import * as regions from './regions.js';
  *
  * 2 — elements with no visible footprint, and containers holding two or more
  *     others, no longer enter identity: only one sensor can see either.
+ * 3 — a chrome label must be a name: at least two letters, and not a URL. A
+ *     browser's address bar put "== example.com" into a screen's identity, so
+ *     a different page read as a different screen, and OCR's ":" and "+" read
+ *     off icons were identities of their own.
  */
-export const TOKEN_RULES_VERSION = 2;
+export const TOKEN_RULES_VERSION = 3;
 
 /** Frames are quantised to this, so sub-pixel drift and a nudged row do not matter. */
 export const GRID = 24;
@@ -97,12 +101,36 @@ const MONTHS = /\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\b/i;
 const WEEKDAYS = /\b(mon|tue|wed|thu|fri|sat|sun)[a-z]*day?\b/i;
 const DATE_LIKE = /\d{1,4}[/.-]\d{1,2}([/.-]\d{1,4})?|\b\d{1,2}:\d{2}\b/;
 
+/**
+ * A URL is the most volatile thing a nav bar can hold.
+ *
+ * Measured on Android, where a browser's address bar is chrome by every
+ * structural test there is: the screen's identity contained `"== example.com"`,
+ * so the same browser on a different page was a different screen, and every
+ * route through it broke on navigation. The `==` is OCR reading the lock icon.
+ *
+ * Matched after stripping the punctuation OCR decorates it with, and only when
+ * the whole label is the address — a sentence that happens to mention a domain
+ * is still a sentence.
+ */
+const URL_LIKE = /^(https?:\/\/|www\.)|^[a-z0-9][a-z0-9-]*(\.[a-z0-9-]+)*\.[a-z]{2,}(\/\S*)?$/i;
+
+/** How many letters a name has to have. One is a glyph, not a name. */
+const NAME_MIN_LETTERS = 2;
+
 export function isVolatileLabel(label) {
   const text = String(label ?? '').trim();
   if (!text) return true;
   if (MONTHS.test(text) || WEEKDAYS.test(text) || DATE_LIKE.test(text)) return true;
+  // Strip what OCR hangs off an icon before asking whether the rest is an
+  // address: the observed label was `== example.com`.
+  const bare = text.replace(/^[^\p{L}\p{N}]+/u, '').replace(/[^\p{L}\p{N}/]+$/u, '');
+  if (URL_LIKE.test(bare)) return true;
   const letters = (text.match(/\p{L}/gu) ?? []).length;
   const digits = (text.match(/\p{N}/gu) ?? []).length;
+  // A label with no word in it is not a name for anything. OCR reads `:`, `+`,
+  // `...` and `—` off icons, and each of those became an identity of its own.
+  if (letters < NAME_MIN_LETTERS) return true;
   // Mostly digits: a count, a price, a phone number, an ID. "1020" and
   // "+1 (111) 111-1111" are both this; "Assets" is not.
   return digits > 0 && digits >= letters;

package/src/index.js

@@ -19,7 +19,7 @@ import * as graph from './graph.js';
 import * as matching from './matching.js';
 import * as refs from './refs.js';
 import * as screenmap from './screenmap.js';
-import { resolveDevice, resize, screenshot } from './simctl.js';
+import { capabilitiesFor, resolveDevice, resize, screenshot } from './platform/index.js';
 import * as store from './store.js';
 
 const HERE = path.dirname(fileURLToPath(import.meta.url));
@@ -186,15 +186,24 @@ export function fallbackReason(udid) {
 }
 
 /**
- * Start whichever engine was asked for.
+ * Start whichever engine was asked for, out of the ones this device's platform
+ * has.
  *
- * simframed unless told otherwise: it reads the framebuffer directly and is
- * roughly thirty times faster per frame. The simctl loop stays reachable with
- * `engine: 'simctl'`, and is used automatically when the daemon cannot be
- * built — a machine with no Swift toolchain still has to work.
+ * On iOS that is simframed unless told otherwise: it reads the framebuffer
+ * directly and is roughly thirty times faster per frame, and the screenshot
+ * loop stays reachable with `engine: 'screenshot'` for a machine with no Swift
+ * toolchain. On Android the loop is the only engine there is — and asking for
+ * simframed there is refused rather than attempted, because a Swift daemon
+ * built against CoreSimulator has nothing to say to an emulator, and the
+ * failure it produces says nothing useful about why.
  */
 async function startEngine(udid, options) {
-  if ((options.engine ?? 'simframed') === 'simframed') {
+  const supported = capabilitiesFor(udid).captureEngines;
+  const wanted = engine.normalizeEngine(options.engine ?? supported[0]);
+  if (!supported.includes(wanted)) {
+    throw new Error(`this device cannot run the ${wanted} capture engine — it supports ${supported.join(', ')}`);
+  }
+  if (wanted === 'simframed') {
     const built = await engine.ensureBuilt();
     if (built.ok) {
       engineFallbackReason = null;
@@ -206,7 +215,7 @@ async function startEngine(udid, options) {
     recordFallback(udid, engineFallbackReason);
   }
   spawnNodeDaemon(udid, options);
-  return 'simctl';
+  return 'screenshot';
 }
 
 function spawnNodeDaemon(udid, options) {
@@ -289,12 +298,38 @@ export function changeLevel(diff) {
   return 'none';
 }
 
+/** How long a stall has been going on, in words an agent can act on. */
+function stallNote(health) {
+  const forMs = Math.max(0, Date.now() - (health.since ?? Date.now()));
+  const parts = [`capture: stalled — the display surface has been unreadable for ${Math.round(forMs / 1000)}s`];
+  if (health.reattaches) parts.push(`${health.reattaches} re-attach${health.reattaches === 1 ? '' : 'es'} did not help`);
+  if (health.reason) parts.push(String(health.reason).slice(0, 120));
+  // The cure is the user's to apply. Saying so is the difference between an
+  // agent that reports "the simulator is wedged" and one that retries a tap
+  // twenty times because nothing appeared to change.
+  parts.push('only restarting the device is known to cure it');
+  return parts.join('; ');
+}
+
 export function liveness(udid, state) {
   const ageMs = Date.now() - state.capturedAt;
   const { running } = daemonStatus(udid);
+  // A wedged device and a quiet one look identical from the frames alone: both
+  // produce nothing. The difference is that a wedged one is failing reads, and
+  // only the capture loop knows that, so it writes it down.
+  const health = store.captureHealth(udid);
+  const stalled = Boolean(health?.stalled);
   if (!running) {
-    return { ok: false, ageMs, note: 'the capture loop has died; the frame you are looking at is the last one it wrote' };
+    return {
+      ok: false,
+      ageMs,
+      stalled,
+      note: stalled
+        ? `the capture loop has died, and it was stalled before it did — ${stallNote(health)}`
+        : 'the capture loop has died; the frame you are looking at is the last one it wrote',
+    };
   }
+  if (stalled) return { ok: false, ageMs, stalled: true, note: stallNote(health) };
   // Frame age means "stalled" only for a fixed-rate loop.
   //
   // simframed captures on damage, so a screen that is genuinely still produces
@@ -306,9 +341,9 @@ export function liveness(udid, state) {
   // daemon.
   const damageDriven = engine.runningEngine(udid) === 'simframed';
   if (!damageDriven && ageMs > STALE_FRAME_MS) {
-    return { ok: false, ageMs, note: `capture loop is stalled: newest frame is ${ageMs}ms old` };
+    return { ok: false, ageMs, stalled: false, note: `capture loop is stalled: newest frame is ${ageMs}ms old` };
   }
-  return { ok: true, ageMs, note: null };
+  return { ok: true, ageMs, stalled: false, note: null };
 }
 
 /**
@@ -502,7 +537,16 @@ export async function waitFor(
       // a button changing state. Waiting the full timeout for a change that
       // will never be visible turns a 100ms action into a 12s one, so give up
       // early and say so, rather than silently burning the clock.
-      if (!sawChange && Date.now() - startedAt > reactionMs && state.stableForMs >= stableMs) {
+      //
+      // But "nothing changed" is a claim about something observed, and with no
+      // frame captured since this call began, nothing has been. The screenshot
+      // engine idles at 1.5 fps, so a 500 ms reaction window expired before the
+      // first new frame existed: a tap that opened a whole activity was
+      // reported as having no visible effect, and the text meant for the field
+      // it opened was typed into nothing. Damage-driven capture on iOS hid this
+      // by being fast.
+      const observedSomething = state.seq - startSeq >= 1;
+      if (!sawChange && observedSomething && Date.now() - startedAt > reactionMs && state.stableForMs >= stableMs) {
         return done(false, { noVisibleChange: true });
       }
 
@@ -724,6 +768,31 @@ export async function settledState(udid, { settleMs = MEMORY_SETTLE_MS, timeoutM
   return { state, settled: false };
 }
 
+/**
+ * Read the screen with the perception layers pinned, and keep nothing.
+ *
+ * Every other path decides for itself which layers to read, which is right for
+ * doing work and useless for measuring: the question "what is the accessibility
+ * tier worth" needs the same frame read twice, once with it and once without.
+ * `persist: false` so measuring teaches the graph nothing.
+ */
+export async function readScreenWith(deviceQuery, { useAx = true, useOcr = true, options } = {}) {
+  const { device, state } = await ensureDaemon(deviceQuery, options);
+  const udid = device.udid;
+  const geo = await deviceGeometry(udid, state);
+  const entry = await screenmap.build(udid, {
+    hash: state.hash,
+    layoutHash: state.layoutHash,
+    fullFrame: await fullFrameFor(udid, state),
+    density: geo.density,
+    screen: { width: geo.pointWidth, height: geo.pointHeight },
+    useAx,
+    useOcr,
+    persist: false,
+  });
+  return { device, entry, points: { width: geo.pointWidth, height: geo.pointHeight } };
+}
+
 export async function locate(
   deviceQuery,
   query,

package/src/input.js

@@ -3,6 +3,7 @@
 // even available?" before it answers anything else.
 import { execFile } from 'node:child_process';
 import * as control from './control.js';
+import { geometryFor, inputDriverFor } from './platform/index.js';
 import { promisify } from 'node:util';
 
 const run = promisify(execFile);
@@ -20,6 +21,12 @@ let driverCache = null;
  * iOS. idb remains the fallback so a machine without the daemon still works.
  */
 export async function driverFor(udid) {
+  // A platform that carries its own input path answers first: there is no
+  // daemon to ask and no idb to fall back to, and reporting either for an
+  // Android emulator is how doctor came to claim "input driver: idb" about a
+  // tool that has never spoken to one.
+  const own = udid ? inputDriverFor(udid) : null;
+  if (own) return { name: own.id, available: true, version: own.detail, reason: null, viaSocket: false };
   if (udid && control.available(udid)) {
     try {
       const status = await control.status(udid);
@@ -140,6 +147,15 @@ export async function screenInfo(udid, { refresh = false } = {}) {
 }
 
 async function readScreenInfo(udid) {
+  // The platform first, where it can answer at all: on Android it is the only
+  // source, and the alternative is `deviceGeometry`'s last-resort guess, which
+  // is an iPhone's numbers and silently wrong for everything else.
+  try {
+    const geo = await geometryFor(udid);
+    if (geo?.pointWidth && geo?.pointHeight) return geo;
+  } catch {
+    /* the backend could not say; the daemon or idb may still be able to */
+  }
   // Ask the daemon first. It holds the device's own point size and scale, which
   // makes it both authoritative and free — and it means geometry no longer
   // needs idb at all. Going to idb first meant a machine without idb could
@@ -300,6 +316,11 @@ export function centerOf(node) {
 
 export async function tapPoint(udid, x, y, { durationMs } = {}) {
   const point = { x: Math.round(x), y: Math.round(y) };
+  const own = inputDriverFor(udid);
+  if (own) {
+    await own.tap(udid, point.x, point.y, durationMs ? { durationMs } : {});
+    return point;
+  }
   if (control.available(udid)) {
     await control.tap(udid, point.x, point.y, durationMs ? { durationMs } : {});
     return point;
@@ -318,6 +339,15 @@ export async function tapLabel(udid, query, { index, durationMs } = {}) {
 }
 
 export async function typeText(udid, value) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    // No pasteboard on Android (docs/DEFERRED.md), so exact text goes through
+    // the same keystroke path as everything else. `event text` carries
+    // characters rather than key positions, so a non-Latin host layout does not
+    // reinterpret them — which is the reason the pasteboard exists on iOS.
+    await own.text(udid, String(value));
+    return;
+  }
   if (control.available(udid)) {
     // The daemon's paste path carries characters rather than key positions, so
     // it is not reinterpreted by the device's keyboard layout.
@@ -329,6 +359,11 @@ export async function typeText(udid, value) {
 
 /** Key events rather than text: for shortcuts and search-as-you-type. */
 export async function typeKeys(udid, value) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    await own.text(udid, String(value));
+    return;
+  }
   if (control.available(udid)) {
     await control.type(udid, String(value));
     return;
@@ -337,6 +372,11 @@ export async function typeKeys(udid, value) {
 }
 
 export async function pressKey(udid, keycode) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    await own.key(udid, keycode);
+    return;
+  }
   await idb(['ui', 'key', '--udid', udid, String(keycode)]);
 }
 
@@ -363,6 +403,13 @@ export async function resetSession(udid) {
 }
 
 export async function pressButton(udid, name) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    // Android's whole key vocabulary is safe to offer: `input keyevent` takes
+    // names through a public API, so unlike Indigo there is nothing to guess.
+    await own.key(udid, name);
+    return;
+  }
   if (control.available(udid)) {
     try {
       await control.press(udid, String(name).toLowerCase());
@@ -376,6 +423,11 @@ export async function pressButton(udid, name) {
 }
 
 export async function swipe(udid, from, to, { durationMs = 300 } = {}) {
+  const own = inputDriverFor(udid);
+  if (own) {
+    await own.swipe(udid, from, to, { durationMs });
+    return;
+  }
   if (control.available(udid)) {
     await control.swipe(udid, from, to, { durationMs });
     return;

package/src/mcp.js

@@ -14,14 +14,14 @@ import * as actions from './actions.js';
 import * as api from './index.js';
 import * as input from './input.js';
 import * as navigate from './navigate.js';
-import { bootedDevices, PERMISSION_SERVICES } from './simctl.js';
+import { bootedDevices, permissionServices } from './platform/index.js';
 import * as store from './store.js';
 import * as view from './view.js';
 
 const deviceProp = {
   device: {
     type: 'string',
-    description: 'Simulator UDID or name substring. Defaults to the booted simulator.',
+    description: 'Device UDID or name substring — a simulator udid or an emulator serial. Defaults to the booted device.',
   },
 };
 
@@ -180,7 +180,7 @@ const TOOLS = [
   },
   {
     name: 'sim_open_url',
-    description: 'Open a URL or deep link on the simulator — the fastest way to reach a screen when the app has a link for it.',
+    description: 'Open a URL or deep link on the device — the fastest way to reach a screen when the app has a link for it.',
     inputSchema: {
       type: 'object',
       properties: { ...deviceProp, url: { type: 'string' } },
@@ -189,7 +189,7 @@ const TOOLS = [
   },
   {
     name: 'sim_permission',
-    description: `Grant, revoke or reset a privacy permission for an app. Do this instead of tapping the system alert: the alert is not part of the app under test, and its buttons move between iOS versions. Services: ${PERMISSION_SERVICES.join(', ')}.`,
+    description: `Grant, revoke or reset a privacy permission for an app. Do this instead of tapping the system alert: the alert is not part of the app under test, and its buttons move between OS versions. Not every service exists on every platform — the device's own backend refuses one it does not have. Services: ${permissionServices().join(', ')}.`,
     inputSchema: {
       type: 'object',
       properties: {
@@ -304,7 +304,7 @@ const TOOLS = [
   },
   {
     name: 'sim_devices',
-    description: 'List booted iOS simulators that simframe can capture.',
+    description: 'List the booted devices simframe can drive — iOS simulators and Android emulators.',
     inputSchema: { type: 'object', properties: {} },
   },
 ];
@@ -826,7 +826,7 @@ function listStateDirs() {
 
 async function devices() {
   const booted = await bootedDevices();
-  if (!booted.length) return { content: [text('no booted simulators')] };
+  if (!booted.length) return { content: [text('no booted devices')] };
   return {
     content: [text(booted.map((d) => `${d.name} · ${d.runtime} · ${d.udid}`).join('\n'))],
   };