simframe 0.9.0 → 0.10.0

package/README.md

@@ -255,10 +255,11 @@ nav-bar:
   #2 text      201,64     Inbox
 content:
   #3 cell      201,140    Weekly digest
-  #4 cell      201,196    Payment received
+  #4 field     201,196    Search = weekly ~ weekly|
+  #5 switch    201,252    Notifications = 1
 tab-bar:
-  #5 text      62,835     Inbox
-  #6 text      201,835    Settings
+  #6 text      62,835     Inbox
+  #7 text      201,835    Settings
 ```
 
 Region first, because "Inbox" the title and "Inbox" the tab differ only by where
@@ -267,6 +268,18 @@ is a selector: whatever this calls `#3`, the next call can tap as `#3` without
 describing it. A ref is valid only while that screen is showing — used on a
 different screen it refuses rather than tapping whatever now sits there.
 
+`= something` is what the control *contains*, from the accessibility tree, and
+`~ something` is what OCR read off the pixels. Both are printed, and where they
+disagree that is the point: one is authoritative and the other is what is
+actually on screen, and a field mid-edit can legitimately differ. A row with no
+`=` is a control that reports no value, not an empty one.
+
+When the elements were recalled from screen memory rather than looked at just
+now, the header says so and how long ago — `elements recalled from 41s ago —
+pass refresh for what is there now`. Identity is cached on purpose, because a
+list with new rows is the same screen; contents are exactly what changes without
+the screen changing, so the age is worth seeing.
+
 Three ways to name a control, anywhere one is named:
 
 | | |
@@ -542,8 +555,10 @@ simframe frame --out=now.png   # newest frame, native resolution, to a file
 simframe strip --count=6       # contact sheet, for an animation
 simframe doctor --strict       # any degraded layer is a non-zero exit
 simframe escalations           # why simframe still needs a model, by reason
+simframe escalations --session # ...this agent only, not every agent on the device
 simframe hpi                   # speed and accuracy against a human baseline
 simframe baseline record settings-larger-text --runs=5   # record the human
+simframe input reset           # rebuild the HID session, without restarting anything
 simframe start / status / stop [--force] / devices
 simframe ui --device=emulator-5554      # or export SIMFRAME_DEVICE once
 ```
@@ -646,6 +661,16 @@ said a word — the exact failure shape, found by the thing built to catch it.
   dramatically between visits will simply be rebuilt.
 - It speeds up *confirming* a fix, not *locating* one. A bug living in a memo
   comparator or a stale closure is not visible in any frame.
+- A switch is tapped at the centre of its frame, and a switch's frame is the
+  whole row — so the tap lands on the label and the control, which sits at the
+  trailing end, does not move. Use `@x,y` on the control for now. Filed with
+  the measurement in `docs/DEFERRED.md`; it is a role-specific tap point, not a
+  patch at one call site.
+- The simulator's display pipeline stops rendering under rapid app relaunch —
+  about six cycles, reproducibly — and every frame comes back black while
+  `simctl` itself reports success. simframe now says so instead of reading a
+  black screen as a calm one, but it cannot fix it: restarting the device is
+  the cure that always works, and it usually recovers on its own.
 
 ## Roadmap
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simframe",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "mcpName": "io.github.lvlrSajjad/simframe",
   "description": "Always-warm iOS Simulator and Android emulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
   "keywords": [

package/scripts/check-private.mjs

@@ -0,0 +1,143 @@
+#!/usr/bin/env node
+// No third-party app identifiers in this repository. Ever, from anyone.
+//
+// simframe is a general-purpose tool: you install it and Claude Code drives
+// *your* app on the simulator. It has no relationship with any particular app,
+// so no particular app's bundle id belongs in it — not in the source, not in
+// the docs, and not in a secret either. A denylist of specific strings would
+// assume there is one app to protect, which is the wrong shape for this.
+//
+// So the rule is a pattern, not a list, and it needs no configuration at all.
+// Anything shaped like a reverse-DNS bundle id is flagged unless it is one of:
+//
+//   * a platform's own                 com.apple.*, com.android.*, com.google.*
+//   * a documentation placeholder      com.example.*, com.acme.*, com.mycompany.*
+//   * this project's own identifiers
+//
+// That works on a fresh clone, on a fork, and in a pull request from a stranger,
+// which a secret does not. An optional `.private-strings` file (gitignored) or
+// $SIMFRAME_PRIVATE_STRINGS still adds extra patterns for anyone who wants them,
+// but nothing depends on either existing.
+//
+// How this got written: a 282-line field-notes file about a real third-party app
+// was committed here by `git add -A` and pushed, an hour after the first version
+// of this script was written to prevent exactly that. It could not fire, because
+// it was waiting for a denylist nobody had supplied. A guard with a
+// precondition is a guard that is off.
+//
+// Nothing here ever prints a match. It prints the file and the line number, so
+// the output of a failed run is safe to paste into an issue, a CI log, or a
+// conversation with an agent — which is where the last one would have gone.
+import fs from 'node:fs';
+import path from 'node:path';
+import { execFileSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const LIST_FILE = path.join(ROOT, '.private-strings');
+
+/**
+ * Bundle-id-shaped strings, and the ones that are fine.
+ *
+ * The first segment is restricted to real reverse-DNS prefixes, which is what
+ * keeps ordinary property chains out: `res.state.seq`, `registry.paths.dir` and
+ * `import.meta.url` all look exactly like bundle ids until you require the head
+ * to be a TLD.
+ */
+const BUNDLE = /\b(?:com|io|org|net|dev|co|app|me|xyz|uk|de|fr|jp|nl|se|ca|au)\.[A-Za-z][A-Za-z0-9_-]{1,30}(?:\.[A-Za-z][A-Za-z0-9_-]{0,30}){1,3}\b/g;
+
+/** Platform-owned, placeholder, or ours. Anything else is somebody's app. */
+export const ALLOWED = [
+  /^com\.apple\./i,
+  /^com\.android\./i,
+  /^com\.google\./i,
+  /^org\.swift\./i,
+  /^org\.json\./i,
+  /^com\.facebook\./i,     // idb, a reference implementation named in the docs
+  /^com\.example\./i,
+  /^com\.acme\./i,
+  /^com\.mycompany\./i,
+  /^com\.yourcompany\./i,
+  /^io\.github\./i,
+];
+
+export function isAllowedIdentifier(id) {
+  return ALLOWED.some((re) => re.test(id));
+}
+
+/** Extra patterns, for anyone who wants them. Nothing depends on this existing. */
+export function patternsFrom({ env, file } = {}) {
+  const raw = [
+    ...String(env ?? '').split(/[\n,]/),
+    ...String(file ?? '').split(/\n/),
+  ];
+  return [...new Set(
+    raw
+      .map((s) => s.trim())
+      .filter((s) => s && !s.startsWith('#'))
+      // One character would match everything, which is a check that only ever
+      // fails and therefore only ever gets disabled.
+      .filter((s) => s.length >= 3)
+      .map((s) => s.toLowerCase()),
+  )];
+}
+
+/** Which lines of `text` are a problem. Line numbers only, never matches. */
+export function offendingLines(text, patterns = []) {
+  const hits = [];
+  const lines = String(text).split('\n');
+  for (let i = 0; i < lines.length; i += 1) {
+    const lower = lines[i].toLowerCase();
+    const why = [];
+    const extra = patterns.filter((p) => lower.includes(p)).length;
+    if (extra) why.push(`${extra} denied pattern(s)`);
+    const ids = (lines[i].match(BUNDLE) ?? []).filter((id) => !isAllowedIdentifier(id));
+    // Reported as a count and a shape, never as the identifier: knowing which
+    // app leaked is worth less to a bug report than not restating it.
+    if (ids.length) why.push(`${ids.length} third-party bundle id(s)`);
+    if (why.length) hits.push({ line: i + 1, why: why.join(', ') });
+  }
+  return hits;
+}
+
+function trackedFiles() {
+  return execFileSync('git', ['ls-files', '-z'], { cwd: ROOT, encoding: 'utf8' })
+    .split('\0')
+    .filter(Boolean);
+}
+
+function main() {
+  const patterns = patternsFrom({
+    env: process.env.SIMFRAME_PRIVATE_STRINGS,
+    file: fs.existsSync(LIST_FILE) ? fs.readFileSync(LIST_FILE, 'utf8') : '',
+  });
+  const files = trackedFiles();
+  let failed = 0;
+  for (const rel of files) {
+    const full = path.join(ROOT, rel);
+    let text;
+    try {
+      const stat = fs.statSync(full);
+      if (!stat.isFile() || stat.size > 4_000_000) continue;
+      text = fs.readFileSync(full, 'utf8');
+    } catch {
+      continue;
+    }
+    // A NUL byte means this is not text, and a substring hit in it is noise.
+    if (text.includes('\0')) continue;
+    for (const hit of offendingLines(text, patterns)) {
+      console.error(`check-private: ${rel}:${hit.line} — ${hit.why}`);
+      failed += 1;
+    }
+  }
+  console.log(`check-private: ${files.length} tracked file(s); no third-party bundle ids`
+    + (patterns.length ? `, plus ${patterns.length} local pattern(s)` : '')
+    + ` — ${failed} line(s) flagged`);
+  if (failed) {
+    console.error('check-private: nothing above prints the match itself. '
+      + 'A third-party app identifier does not belong in a general-purpose tool.');
+    process.exitCode = 1;
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) main();

package/scripts/eval-perception.mjs

@@ -0,0 +1,248 @@
+#!/usr/bin/env node
+// The perception eval harness. Deferred since Phase 5; four things wait on it.
+//
+// What it is, and the design decision behind it. The obvious harness replays
+// stored frames through the perception path and diffs the element lists, which
+// is what Phase 13 step 5 describes. That harness cannot exist: the element
+// list is the accessibility tree fused with OCR, the tree is not in the frame,
+// and OCR runs in the daemon against a live framebuffer. A frame on disk is
+// half the input.
+//
+// So the split is different and, for what actually needs gating, better. The
+// machine records the *input* — the fused element list for a screen, exactly as
+// perception produced it. A person authors the *expected output*. Everything
+// downstream of the element list is a pure function, so the check runs offline,
+// deterministically, with no simulator and no daemon:
+//
+//   resolution   matching.resolve(targets, query)  — which element a query picks
+//   identity     fingerprint.tokens(targets)       — which screen this is
+//   change       analyze.signatureDiff(a, b)       — whether a frame moved
+//
+// Those three are precisely the thresholds every blocked item wants to change.
+// What it does *not* cover is whether perception found the elements at all,
+// which is inherently live and stays with eval-fingerprint.mjs and the
+// integration job. Said plainly here rather than implied, because a harness
+// that is trusted for more than it measures is worse than no harness.
+//
+// Two rules from this repo:
+//   - The gate is an exit code. Nothing is verified through a pipe: this
+//     project has already shipped a check that could not fail because
+//     `node script.mjs | tail` reports tail's status.
+//   - A fixture with no authored expectations is reported as unauthored, never
+//     counted as a pass. An empty test suite is green.
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as fingerprint from '../src/fingerprint.js';
+import * as matching from '../src/matching.js';
+import * as analyze from '../src/analyze.js';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const DIR = path.join(ROOT, 'test', 'perception', 'screens');
+const arg = (n, d = null) => {
+  const hit = process.argv.find((a) => a.startsWith(`--${n}=`));
+  return hit ? hit.slice(n.length + 3) : d;
+};
+const has = (n) => process.argv.includes(`--${n}`);
+
+export function fixtures(dir = DIR, only = null) {
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir)
+    .filter((f) => f.endsWith('.json'))
+    .filter((f) => (only ? f.includes(only) : true))
+    .sort()
+    .map((f) => ({ file: f, ...JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) }));
+}
+
+/**
+ * Check one screen's authored expectations against the pure layers.
+ *
+ * Returns findings rather than printing, so the same function backs the CLI and
+ * the unit tests — and so a finding can be counted without being formatted.
+ */
+export function checkScreen(fx) {
+  const findings = [];
+  const screen = fx.points ?? null;
+  const targets = fx.targets ?? [];
+  const expect = fx.expect ?? {};
+  const authored = (expect.resolutions?.length ?? 0)
+    + (expect.ambiguous?.length ?? 0)
+    + (expect.none?.length ?? 0)
+    + (fx.frame_pairs?.length ?? 0)
+    + (expect.identity ? 1 : 0);
+
+  for (const r of expect.resolutions ?? []) {
+    const out = matching.resolve(targets, r.query, { screen });
+    if (out.status !== 'ok') {
+      findings.push({
+        kind: 'resolution',
+        query: r.query,
+        want: r.label,
+        got: out.status === 'ambiguous'
+          ? `ambiguous between ${out.alternatives.map((a) => a.label).join(', ')}`
+          : 'nothing resolved',
+      });
+      continue;
+    }
+    const got = out.target.label ?? '(icon-only)';
+    // Compared on the label because that is what the author can see and mean.
+    // Coordinates would make a fixture break every time a row moved a point.
+    if (got !== r.label) {
+      findings.push({ kind: 'resolution', query: r.query, want: r.label, got: `"${got}" at ${out.target.x},${out.target.y}` });
+    }
+  }
+
+  // The other half of the contract, and the half with the wrong-tap risk in it:
+  // "when two things answer equally well it says so rather than guessing".
+  for (const q of expect.ambiguous ?? []) {
+    const out = matching.resolve(targets, typeof q === 'string' ? q : q.query, { screen });
+    if (out.status !== 'ambiguous') {
+      findings.push({
+        kind: 'should-ask',
+        query: typeof q === 'string' ? q : q.query,
+        want: 'ambiguous',
+        got: out.status === 'ok' ? `picked "${out.target.label}" at score ${out.score}` : 'nothing resolved',
+      });
+    }
+  }
+
+  for (const q of expect.none ?? []) {
+    const out = matching.resolve(targets, q, { screen });
+    if (out.status !== 'none') {
+      findings.push({
+        kind: 'should-find-nothing',
+        query: q,
+        want: 'none',
+        got: out.status === 'ok' ? `picked "${out.target.label}"` : 'ambiguous',
+      });
+    }
+  }
+
+  // Identity drift. A token-rule change that silently alters a screen's
+  // structural hash discards every stored map and graph for it, and the failure
+  // is invisible — an old hash is a well-formed hash that matches nothing.
+  if (expect.identity && screen) {
+    const now = fingerprint.fingerprint(targets, screen);
+    if (now.hash !== expect.identity.hash) {
+      const similarity = fingerprint.similarity(now.tokens, expect.identity.tokens ?? []);
+      findings.push({
+        kind: 'identity',
+        query: '(structural hash)',
+        want: `${expect.identity.hash?.slice(0, 12)} (${expect.identity.tokens?.length ?? 0} tokens)`,
+        got: `${now.hash?.slice(0, 12)} (${now.tokens.length} tokens), similarity ${similarity.toFixed(2)}`,
+      });
+    }
+  }
+
+  // Frame pairs, for the change detector. `changed: true` means an action that
+  // a person would call visible — a switch flipping, a radio dot moving.
+  for (const pair of fx.frame_pairs ?? []) {
+    const a = analyze.hexToSignature(pair.after);
+    const b = analyze.hexToSignature(pair.before);
+    // Two different questions about the same pair of frames: has the *screen*
+    // changed (the mean, which drives stillness) and has a *control* changed
+    // (the largest single region, which drives the verdict). A switch flip
+    // answers no to the first and yes to the second, which is the whole reason
+    // both exist.
+    const diff = pair.per_cell ? analyze.maxCellDelta(a, b) : analyze.signatureDiff(a, b);
+    const seen = diff > (pair.threshold ?? 0.004);
+    if (seen !== Boolean(pair.changed)) {
+      findings.push({
+        kind: 'change',
+        query: pair.note ?? '(frame pair)',
+        want: pair.changed ? 'a visible change' : 'no change',
+        got: `${pair.per_cell ? 'max cell' : 'mean'} ${diff.toFixed(5)} against a threshold of ${pair.threshold ?? 0.004}`,
+      });
+    }
+  }
+
+  return { authored, findings };
+}
+
+async function record() {
+  const api = await import('../src/index.js');
+  const device = arg('device');
+  const name = arg('name');
+  if (!name) throw new Error('--record needs --name=<app>/<screen>');
+  const id = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
+  const entry = id.entry ?? {};
+  const out = {
+    app: name.split('/')[0],
+    screen: name.split('/').slice(1).join('/') || name,
+    note: arg('note') ?? null,
+    recorded_at: new Date().toISOString(),
+    points: id.points,
+    // The recorded input. Trimmed to what the pure layers read, so a fixture
+    // is reviewable by a person rather than a wall of machine state.
+    targets: (entry.targets ?? []).map((t) => ({
+      label: t.label ?? null,
+      value: t.value ?? null,
+      type: t.type ?? null,
+      x: t.x, y: t.y,
+      frame: t.frame ?? null,
+      region: t.region ?? null,
+      source: t.source ?? null,
+      aliases: t.aliases ?? undefined,
+      navSlot: t.navSlot ?? undefined,
+      enabled: t.enabled ?? undefined,
+      selected: t.selected ?? undefined,
+    })),
+    expect: {
+      identity: { hash: entry.structuralHash, tokens: entry.structuralTokens ?? [] },
+      // Authored by hand. The machine records what perception saw; a person
+      // says what it should mean. Deriving these from current behaviour would
+      // bake today's bugs in as the specification.
+      resolutions: [],
+      ambiguous: [],
+      none: [],
+    },
+  };
+  fs.mkdirSync(DIR, { recursive: true });
+  const file = path.join(DIR, `${name.replace(/\//g, '__')}.json`);
+  fs.writeFileSync(file, `${JSON.stringify(out, null, 2)}\n`);
+  console.log(`recorded ${out.targets.length} element(s) from ${name} -> ${path.relative(ROOT, file)}`);
+  console.log('  now author expect.resolutions / expect.ambiguous / expect.none by hand');
+}
+
+function check() {
+  const all = fixtures(DIR, arg('only'));
+  if (!all.length) {
+    console.error('check-perception: no fixtures. Record some with --record --name=<app>/<screen>');
+    process.exitCode = 1;
+    return;
+  }
+  let failed = 0;
+  let unauthored = 0;
+  let checks = 0;
+  const apps = new Set();
+  for (const fx of all) {
+    apps.add(fx.app);
+    const { authored, findings } = checkScreen(fx);
+    checks += authored;
+    if (!authored) {
+      unauthored += 1;
+      console.log(`  ??  ${fx.app}/${fx.screen}  recorded, no expectations authored`);
+      continue;
+    }
+    if (!findings.length) {
+      console.log(`  ok  ${fx.app}/${fx.screen}  ${authored} expectation(s)`);
+      continue;
+    }
+    failed += findings.length;
+    console.log(`FAIL  ${fx.app}/${fx.screen}`);
+    for (const f of findings) {
+      console.log(`        ${f.kind}: ${f.query}\n          want ${f.want}\n          got  ${f.got}`);
+    }
+  }
+  console.log(`\n${all.length} screen(s) across ${apps.size} app(s), ${checks} expectation(s), ${failed} failure(s)`
+    + (unauthored ? `, ${unauthored} unauthored` : ''));
+  // An unauthored fixture is not a pass. A suite that counts recordings as
+  // successes is a suite that goes green by adding files.
+  if (failed || unauthored) process.exitCode = 1;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  if (has('record')) await record();
+  else if (has('list')) for (const f of fixtures()) console.log(`${f.app}/${f.screen}  ${f.targets?.length ?? 0} elements`);
+  else check();
+}