simframe 0.4.2 → 0.6.0-rc.1

package/src/control.js

@@ -0,0 +1,77 @@
+// Client for the simframed control socket.
+//
+// One JSON object per line over a per-device Unix socket. The socket is mode
+// 0600, so reachability is the whole authorisation model: if you can open it,
+// you are the user who owns the simulator.
+import fs from 'node:fs';
+import net from 'node:net';
+import path from 'node:path';
+import * as store from './store.js';
+
+export function socketPath(udid) {
+  return path.join(store.deviceDir(udid), 'control.sock');
+}
+
+/** Whether a daemon is listening for this device. */
+export function available(udid) {
+  const meta = store.readJson(path.join(store.deviceDir(udid), 'meta.json'));
+  if (!meta || !store.isProcessAlive(meta.pid)) return false;
+  try {
+    return fs.statSync(socketPath(udid)).isSocket();
+  } catch {
+    return false;
+  }
+}
+
+export function request(udid, payload, { timeoutMs = 30_000 } = {}) {
+  return new Promise((resolve, reject) => {
+    const socket = net.createConnection(socketPath(udid));
+    let buffer = '';
+    let settled = false;
+    const finish = (fn, value) => {
+      if (settled) return;
+      settled = true;
+      socket.destroy();
+      fn(value);
+    };
+    socket.setTimeout(timeoutMs, () => finish(reject, new Error('control socket timed out')));
+    socket.on('connect', () => socket.write(`${JSON.stringify(payload)}\n`));
+    socket.on('data', (chunk) => {
+      buffer += chunk;
+      const line = buffer.indexOf('\n');
+      if (line < 0) return;
+      try {
+        const response = JSON.parse(buffer.slice(0, line));
+        if (response.ok === false) finish(reject, new Error(response.error || 'request failed'));
+        else finish(resolve, response);
+      } catch (err) {
+        finish(reject, err);
+      }
+    });
+    socket.on('error', (err) => {
+      finish(
+        reject,
+        err.code === 'ENOENT' || err.code === 'ECONNREFUSED'
+          ? new Error('no simframed daemon is listening for this device')
+          : err,
+      );
+    });
+  });
+}
+
+export const tap = (udid, x, y, opts = {}) => request(udid, { action: 'tap', x, y, ...opts });
+export const swipe = (udid, from, to, opts = {}) =>
+  request(udid, { action: 'swipe', x1: from.x, y1: from.y, x2: to.x, y2: to.y, ...opts });
+export const type = (udid, text) => request(udid, { action: 'type', text });
+export const paste = (udid, text) => request(udid, { action: 'paste', text });
+export const press = (udid, button) => request(udid, { action: 'press', button });
+export const status = (udid) => request(udid, { action: 'status' });
+export const resetInput = (udid) => request(udid, { action: 'resetInput' });
+export const longPress = (udid, x, y, opts = {}) => request(udid, { action: 'longPress', x, y, ...opts });
+export const drag = (udid, from, to, opts = {}) =>
+  request(udid, { action: 'drag', x1: from.x, y1: from.y, x2: to.x, y2: to.y, ...opts });
+export const launch = (udid, bundleId, opts = {}) => request(udid, { action: 'launch', bundleId, ...opts });
+export const terminate = (udid, bundleId) => request(udid, { action: 'terminate', bundleId });
+export const openUrl = (udid, url) => request(udid, { action: 'openUrl', url });
+export const permission = (udid, permissionAction, service, bundleId) =>
+  request(udid, { action: 'permission', permissionAction, service, bundleId });

package/src/daemon.js

@@ -16,7 +16,14 @@ import { isBootedSync, resize, screenshot } from './simctl.js';
 
 // Bump whenever the shape of state.json changes, so an upgraded client retires
 // a capture loop left running by an older install instead of misreading it.
-export const STATE_VERSION = 5;
+// MUST equal SimframeCore.FrameStore.stateVersion in the Swift daemon. When
+// these drifted — Node on 5, Swift writing 6 — every single CLI command judged
+// the live daemon stale and spawned a replacement: 993 "superseded by another
+// capture loop" lines in one log. Capture still worked, so nothing looked
+// wrong, but each command lost the previous daemon's history, which silently
+// broke `recall`, `state --since`, `wait --since` and every timing measured
+// through a flow. A unit test asserts these two constants match.
+export const STATE_VERSION = 6;
 
 export const DEFAULTS = {
   fps: 4,

package/src/engine.js

@@ -0,0 +1,99 @@
+// Chooses and starts the capture engine.
+//
+// Two exist: `simframed`, a Swift daemon that reads the framebuffer directly,
+// and `simctl`, the original loop that shells out for each screenshot. The
+// daemon is the default because it is roughly thirty times faster, but the old
+// loop stays reachable — a machine without a Swift toolchain, or an Xcode
+// version where a private symbol has moved, still needs to work.
+import { execFile, spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+import * as store from './store.js';
+
+const run = promisify(execFile);
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const PACKAGE = path.join(HERE, '..', 'native', 'simframed');
+const BINARY = path.join(PACKAGE, '.build', 'release', 'simframed');
+
+export const ENGINES = ['simframed', 'simctl'];
+
+export function binaryPath() {
+  return BINARY;
+}
+
+/** Newest mtime across the Swift sources, so a stale binary is rebuilt. */
+function sourceMtime() {
+  const roots = [path.join(PACKAGE, 'Sources'), path.join(PACKAGE, 'Package.swift')];
+  let newest = 0;
+  const walk = (p) => {
+    let stat;
+    try {
+      stat = fs.statSync(p);
+    } catch {
+      return;
+    }
+    if (stat.isDirectory()) {
+      for (const name of fs.readdirSync(p)) walk(path.join(p, name));
+      return;
+    }
+    if (stat.mtimeMs > newest) newest = stat.mtimeMs;
+  };
+  for (const r of roots) walk(r);
+  return newest;
+}
+
+export function status() {
+  const haveSource = fs.existsSync(path.join(PACKAGE, 'Package.swift'));
+  const haveBinary = fs.existsSync(BINARY);
+  const stale = haveBinary && haveSource && fs.statSync(BINARY).mtimeMs < sourceMtime();
+  return { haveSource, haveBinary, stale, binary: BINARY };
+}
+
+/**
+ * Make sure the daemon binary exists and is current.
+ * @returns {Promise<{ok: boolean, built: boolean, reason?: string}>}
+ */
+export async function ensureBuilt({ rebuild = false } = {}) {
+  const state = status();
+  if (!state.haveSource) {
+    return { ok: false, built: false, reason: 'the simframed sources are not in this install' };
+  }
+  if (state.haveBinary && !state.stale && !rebuild) return { ok: true, built: false };
+  try {
+    // Release, because a debug build is several times slower per frame and
+    // this binary's whole purpose is latency.
+    await run('swift', ['build', '-c', 'release', '--package-path', PACKAGE], { timeout: 300_000 });
+    return { ok: fs.existsSync(BINARY), built: true };
+  } catch (err) {
+    const detail = (err.stderr || err.message || '').split('\n').filter(Boolean).slice(-2).join(' ');
+    return {
+      ok: false,
+      built: false,
+      reason:
+        err.code === 'ENOENT'
+          ? 'swift is not installed, so the daemon cannot be built (install Xcode command line tools)'
+          : `building simframed failed: ${detail}`,
+    };
+  }
+}
+
+/** Start the Swift daemon detached, the way the Node loop is started. */
+export function spawnDaemon(udid, { maxDim, minIntervalMs, idleExitMs } = {}) {
+  const args = ['run', `--udid=${udid}`];
+  if (maxDim != null) args.push(`--max-dim=${maxDim}`);
+  if (minIntervalMs != null) args.push(`--min-interval-ms=${minIntervalMs}`);
+  if (idleExitMs != null) args.push(`--idle-exit-ms=${idleExitMs}`);
+  const log = fs.openSync(path.join(store.deviceDir(udid), 'simframed.log'), 'a');
+  const child = spawn(BINARY, args, { detached: true, stdio: ['ignore', log, log] });
+  child.unref();
+  return child;
+}
+
+/** Which engine wrote the state we are reading, according to meta.json. */
+export function runningEngine(udid) {
+  const meta = store.readJson(path.join(store.deviceDir(udid), 'meta.json'));
+  if (!meta || !store.isProcessAlive(meta.pid)) return null;
+  return meta.options?.engine === 'simframed' ? 'simframed' : 'simctl';
+}

package/src/fingerprint.js

@@ -0,0 +1,183 @@
+// Identity, as distinct from change.
+//
+// The pixel dHash answers "did this move?", which is a question about pixels
+// and which it answers well. It is a poor answer to "is this the same screen?",
+// because content is pixels: a list whose rows changed drifts as far as a
+// different screen does. Measured, same-screen revisits reached 62 bits against
+// a different-screen floor of 74 — no threshold separates those.
+//
+// This fingerprints layout instead. Two screens are the same when the same
+// kinds of thing sit in the same places, whatever they currently say.
+import crypto from 'node:crypto';
+import * as regions from './regions.js';
+
+/** Frames are quantised to this, so sub-pixel drift and a nudged row do not matter. */
+export const GRID = 24;
+
+/** Regions whose labels identify the screen rather than describe its contents. */
+const CHROME = new Set(['nav-bar', 'tab-bar']);
+
+/**
+ * How wide a tab label can be before it is not a tab label.
+ *
+ * The region bands are positional, so anything low enough on the screen lands
+ * in `tab-bar` — including page content sitting just above the real tabs. One
+ * app put a date banner there, and because chrome labels go into the
+ * fingerprint, that screen's identity contained "sep 08, 2026" and would have
+ * become a different screen at midnight. Every stored map, node and route
+ * touching it would have broken overnight.
+ *
+ * A tab bar divides its width between its tabs, so a tab's label is a fraction
+ * of the screen: measured on that app, real tab labels ran 24-72 px against the
+ * banner's 144 px on a 402 px screen. The element still contributes its shape
+ * to the fingerprint — presence is structure — it just stops contributing text.
+ */
+const TAB_LABEL_MAX_WIDTH_FRACTION = 0.3;
+
+const quantise = (v) => Math.round((v ?? 0) / GRID);
+
+/** Coarse role, so "Button" and "AXButton" and an OCR-inferred button agree. */
+export function roleOf(target) {
+  const t = String(target.type ?? '').toLowerCase();
+  if (/button/.test(t)) return 'button';
+  if (/textfield|textview|searchfield|field/.test(t)) return 'field';
+  if (/switch|toggle|checkbox/.test(t)) return 'switch';
+  if (/cell|row/.test(t)) return 'cell';
+  if (/link/.test(t)) return 'link';
+  if (/image|icon/.test(t)) return 'image';
+  if (/statictext|text|label/.test(t)) return 'text';
+  if (/group|other|generic/.test(t)) return 'group';
+  return t || 'unknown';
+}
+
+/**
+ * One, or several.
+ *
+ * Finer buckets were tried and are worse: a list that grows from three rows to
+ * seven crosses a 2-4 / 5+ boundary and changes identity, which is exactly the
+ * instability the bucketing existed to prevent. "A group of these lives here"
+ * is the stable fact; how many there are today is content.
+ */
+function bucket(n) {
+  return n <= 1 ? '1' : 'many';
+}
+
+const normLabel = (s) => String(s ?? '').toLowerCase().replace(/\s+/g, ' ').trim().slice(0, 40);
+
+/**
+ * A value, not a name.
+ *
+ * Phase 6d removed a date banner from one screen's identity after that
+ * fingerprint would have expired at midnight. It came back through a different
+ * door: measured across twenty screens of a real app, three carried content in
+ * their identity because the positional region bands had called it chrome — a
+ * store address, a phone number, and a nav title reading "Tuesday, September 8".
+ * That last one is a screen whose identity has until midnight to live.
+ *
+ * The band misclassification is the root cause and is fixed by clustering, not
+ * by another threshold (docs/DEFERRED.md). What can be fixed here without
+ * guessing at geometry is the narrower question: is this text a name for the
+ * screen, or is it today's value? A name is words. A date, a phone number, a
+ * price and a bare count are not, and every one of them changes while the
+ * screen stays the same screen.
+ */
+const MONTHS = /\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\b/i;
+const WEEKDAYS = /\b(mon|tue|wed|thu|fri|sat|sun)[a-z]*day?\b/i;
+const DATE_LIKE = /\d{1,4}[/.-]\d{1,2}([/.-]\d{1,4})?|\b\d{1,2}:\d{2}\b/;
+
+export function isVolatileLabel(label) {
+  const text = String(label ?? '').trim();
+  if (!text) return true;
+  if (MONTHS.test(text) || WEEKDAYS.test(text) || DATE_LIKE.test(text)) return true;
+  const letters = (text.match(/\p{L}/gu) ?? []).length;
+  const digits = (text.match(/\p{N}/gu) ?? []).length;
+  // Mostly digits: a count, a price, a phone number, an ID. "1020" and
+  // "+1 (111) 111-1111" are both this; "Assets" is not.
+  return digits > 0 && digits >= letters;
+}
+
+/**
+ * The canonical tokens this screen is made of.
+ *
+ * Deliberately excluded: the status bar (a clock is not identity), everything
+ * inside the keyboard when one is up (it is the same keyboard on every screen),
+ * and the text of anything in the content region (that is the content).
+ */
+export function tokens(targets, screen) {
+  if (!screen?.width || !screen?.height) return { tokens: [], keyboard: false };
+  const keyboardTop = regions.detectKeyboardTop(targets, screen);
+  const groups = new Map();
+
+  for (const t of targets) {
+    const frame = t.frame ?? { x: t.x, y: t.y, width: 0, height: 0 };
+    // Off-screen elements are not part of what this screen looks like.
+    if (frame.y + (frame.height ?? 0) <= 0 || frame.y >= screen.height) continue;
+    const region = t.region ?? regions.regionFor(frame, screen, { keyboardTop });
+    if (region === 'status-bar') continue;
+    if (keyboardTop != null && frame.y >= keyboardTop) continue;
+
+    const role = roleOf(t);
+    // Group by what a thing IS and how big it is, not where it is. Repeated
+    // siblings — the rows of a list — differ only in position, and including
+    // position in the key makes a four-row list a different screen from a
+    // three-row one.
+    const parts = [role, region];
+    // Where in the nav bar a thing sits is structure, not content — and it is
+    // what tells a title apart from a button that happens to be up there.
+    if (CHROME.has(region) && t.navSlot) parts.push(`@${t.navSlot}`);
+    parts.push(`w${quantise(frame.width)}`, `h${quantise(frame.height)}`);
+    // Chrome labels are the only text that survives: two list screens with
+    // identical structure differ by their title, and nothing else says so. But
+    // only where the element is plausibly chrome — a nav bar has slots, and a
+    // tab label is narrow; content that merely fell into the band is not a name.
+    const labelWorthKeeping = CHROME.has(region)
+      && t.label
+      && !isVolatileLabel(t.label)
+      && (region !== 'tab-bar' || (frame.width ?? 0) <= screen.width * TAB_LABEL_MAX_WIDTH_FRACTION);
+    if (labelWorthKeeping) parts.push(`"${normLabel(t.label)}"`);
+    const key = parts.join(':');
+    const group = groups.get(key) ?? { count: 0, x: quantise(frame.x), y: quantise(frame.y) };
+    group.count += 1;
+    // Anchor the group at its topmost member, which is stable as a list grows.
+    if (quantise(frame.y) < group.y) {
+      group.x = quantise(frame.x);
+      group.y = quantise(frame.y);
+    }
+    groups.set(key, group);
+  }
+
+  const out = [...groups.entries()]
+    .map(([key, g]) => (g.count > 1
+      // A repeated group is identified by its anchor and how many of it there
+      // roughly are, never by an exact count.
+      ? `${key}:x${g.x}:y${g.y}#${bucket(g.count)}`
+      : `${key}:x${g.x}:y${g.y}#1`))
+    .sort();
+  return { tokens: out, keyboard: keyboardTop != null };
+}
+
+export function hashTokens(list) {
+  return crypto.createHash('sha256').update(list.join('\n')).digest('hex').slice(0, 32);
+}
+
+/** Structural fingerprint of a screen, plus the tokens it was built from. */
+export function fingerprint(targets, screen) {
+  const { tokens: list, keyboard } = tokens(targets, screen);
+  return { hash: hashTokens(list), tokens: list, keyboard, count: list.length };
+}
+
+/**
+ * How alike two token sets are, 0 to 1.
+ *
+ * Jaccard rather than Hamming: the sets are of different sizes when an optional
+ * element appears — a badge, a banner — and that should cost a little, not
+ * everything.
+ */
+export function similarity(a = [], b = []) {
+  if (!a.length && !b.length) return 1;
+  const setA = new Set(a);
+  const setB = new Set(b);
+  let shared = 0;
+  for (const token of setA) if (setB.has(token)) shared += 1;
+  return shared / (setA.size + setB.size - shared);
+}