simframe 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +181 -2
  2. package/data/vocabulary/en.json +148 -0
  3. package/native/ocr.swift +13 -1
  4. package/native/rank.swift +87 -0
  5. package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +72 -4
  6. package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +27 -0
  7. package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +4 -0
  8. package/native/simframed/Sources/SimframeCore/CaptureRecovery.swift +19 -0
  9. package/native/simframed/Sources/simframed/main.swift +33 -3
  10. package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +41 -0
  11. package/native/supervise.swift +216 -0
  12. package/package.json +4 -1
  13. package/scripts/check-package.mjs +22 -2
  14. package/scripts/check-private.mjs +9 -0
  15. package/scripts/ci-integration-local.sh +79 -0
  16. package/scripts/ci-memory.mjs +104 -20
  17. package/scripts/collect-rulings.mjs +312 -0
  18. package/scripts/eval-fingerprint.mjs +100 -23
  19. package/scripts/eval-perception.mjs +33 -0
  20. package/scripts/phase17-corpus.mjs +176 -0
  21. package/scripts/probe-network.mjs +118 -0
  22. package/scripts/soak-capture.mjs +72 -0
  23. package/skills/simframe/SKILL.md +257 -7
  24. package/src/actions.js +1791 -44
  25. package/src/cli.js +216 -14
  26. package/src/control.js +1 -0
  27. package/src/fingerprint.js +43 -1
  28. package/src/graph.js +136 -7
  29. package/src/index.js +252 -14
  30. package/src/input.js +66 -3
  31. package/src/localhelper.js +161 -0
  32. package/src/matching.js +64 -1
  33. package/src/mcp.js +333 -32
  34. package/src/metrics.js +148 -3
  35. package/src/ocr.js +18 -1
  36. package/src/planner.js +195 -0
  37. package/src/platform/android.js +25 -1
  38. package/src/platform/index.js +11 -1
  39. package/src/platform/ios.js +25 -1
  40. package/src/png.js +26 -0
  41. package/src/refs.js +51 -8
  42. package/src/regions.js +215 -1
  43. package/src/screenmap.js +89 -9
  44. package/src/supervisor.js +161 -0
  45. package/src/view.js +375 -11
  46. package/src/vocabulary.js +134 -0
  47. package/src/wrote.js +136 -0
package/src/index.js CHANGED
@@ -5,6 +5,7 @@ import path from 'node:path';
5
5
  import { fileURLToPath } from 'node:url';
6
6
  import { DEFAULTS, STATE_VERSION } from './daemon.js';
7
7
  import * as engine from './engine.js';
8
+ import * as regions from './regions.js';
8
9
  import { decodePng, encodePng, scaleBitmap } from './png.js';
9
10
  import {
10
11
  REGION_COLS,
@@ -302,6 +303,18 @@ export const STALE_FRAME_MS = 2500;
302
303
  */
303
304
  export const MEMORY_SETTLE_MS = 250;
304
305
 
306
+ /**
307
+ * How well a relabelled ref must match before it is acted on.
308
+ *
309
+ * Above `matching.MINIMUM_SCORE` (0.45) on purpose: that floor is for a label
310
+ * the caller wrote, and this is a label simframe substituted after refusing
311
+ * their `#n`. A fuzzy name match returns `similarity * 0.72` and a prefix match
312
+ * is scaled by its coverage, so neither can reach 0.8 on the name alone — which
313
+ * makes this "the label matched nearly exactly", not a tuned constant. The
314
+ * recovery that made this necessary scored 0.64.
315
+ */
316
+ export const RELABEL_MIN_SCORE = 0.8;
317
+
305
318
  /** Below this a "change" is a clock digit or a caret, not a new screen. */
306
319
  export const MINOR_CHANGE = 0.004;
307
320
  export const MAJOR_CHANGE = 0.03;
@@ -536,9 +549,31 @@ export async function getFrame(deviceQuery, { detail = 'normal', options } = {})
536
549
  let scaledOnRead = false;
537
550
  if (maxDim === 0) {
538
551
  file = state.fullFile;
539
- } else if (maxDim > nativeMax + 8 && fs.existsSync(state.fullFile)) {
552
+ } else if (maxDim > nativeMax + 8) {
553
+ // Asking for more detail than the ring frame holds, so it has to come from
554
+ // a full-resolution frame. The `fs.existsSync(state.fullFile)` this used to
555
+ // require is exactly the condition that fails routinely — retention thins
556
+ // full frames aggressively, and `state.fullFile` names one that is often
557
+ // already gone. When it did, this fell straight through to `latest.png` and
558
+ // returned a 322x700 image while still calling itself detail "high".
559
+ //
560
+ // That was not a small inaccuracy. Every single `sim_look` header in a
561
+ // two-agent field round read `322x700` on a 402x874pt device, so "high
562
+ // (1024px, readable small text)" was interpolating a sub-1x source the
563
+ // whole time. `fullFrameFor` has always known how to find or take a full
564
+ // frame — state named full/9660.png while the directory held eight frames
565
+ // at 1206x2622 — and this path simply never asked it.
566
+ //
567
+ // What this does *not* explain, though it is tempting: the OCR corruption
568
+ // in that round's log (`saaiad@example.com`, `suomit order`). OCR has
569
+ // always gone through `fullFrameFor`, and the daemon's own text recognition
570
+ // reads the live surface at native resolution, so neither was ever looking
571
+ // at the small frame. Those errors are a desktop-width page rendering its
572
+ // labels at a few pixels tall. Fixing what the caller *sees* does not fix
573
+ // what OCR reads, and saying so here keeps the next reader from assuming it did.
574
+ const source = await fullFrameFor(device.udid, state);
540
575
  const out = path.join(p.dir, `read-${maxDim}.png`);
541
- await resize(state.fullFile, out, maxDim);
576
+ await resize(source, out, maxDim);
542
577
  file = out;
543
578
  scaledOnRead = true;
544
579
  } else if (maxDim < nativeMax - 8) {
@@ -550,17 +585,50 @@ export async function getFrame(deviceQuery, { detail = 'normal', options } = {})
550
585
 
551
586
  const png = fs.readFileSync(file);
552
587
  const bmp = pngSize(png);
588
+ // The age must describe the *bytes*, not the state that dates them.
589
+ //
590
+ // These are two different files. `state.json` is written when a frame is
591
+ // recorded; the image is a separate write. A freshly started daemon publishes
592
+ // fresh state while `latest.png` is still the previous session's — which is
593
+ // when a caller's first look of a session happens. Reported, and it is the
594
+ // worst failure this tool can have: a header reading `frame #714 · 84ms old ·
595
+ // still for 17173ms` above an image whose status-bar clock said 6:11 when the
596
+ // real time was 11:10. `sim_ui` was right in the same session because the
597
+ // accessibility tree is read live and in-process; only the image comes from a
598
+ // file, so only the image can be hours stale while the header says otherwise.
599
+ //
600
+ // Taking the *larger* of the two ages cannot overstate freshness. It can
601
+ // overstate staleness by however long the two writes are apart, which is 6ms
602
+ // measured on a healthy daemon — the right direction to be wrong in.
603
+ let fileAgeMs = null;
604
+ try { fileAgeMs = Date.now() - fs.statSync(file).mtimeMs; } catch { /* stat is advisory */ }
605
+ const stateAgeMs = Date.now() - state.capturedAt;
606
+ const ageMs = Number.isFinite(fileAgeMs) ? Math.max(stateAgeMs, fileAgeMs) : stateAgeMs;
553
607
  return {
554
608
  device,
555
609
  state,
556
610
  png,
557
611
  width: bmp.width,
558
612
  height: bmp.height,
559
- ageMs: Date.now() - state.capturedAt,
613
+ ageMs,
614
+ // Said out loud when the image is materially older than the state, because
615
+ // "this picture is not of the screen the rest of this response describes" is
616
+ // not something a caller can work out for themselves.
617
+ frameBehindMs: Number.isFinite(fileAgeMs) && fileAgeMs - stateAgeMs > FRAME_BEHIND_MS
618
+ ? Math.round(fileAgeMs - stateAgeMs)
619
+ : null,
560
620
  scaledOnRead,
561
621
  };
562
622
  }
563
623
 
624
+ /**
625
+ * How far the image may lag the state before it is worth saying so.
626
+ *
627
+ * Measured on a healthy daemon, the two writes land 6ms apart. A second is far
628
+ * outside that and far inside the hours-stale case this exists to catch.
629
+ */
630
+ const FRAME_BEHIND_MS = 1000;
631
+
564
632
  function pngSize(png) {
565
633
  return { width: png.readUInt32BE(16), height: png.readUInt32BE(20) };
566
634
  }
@@ -1068,10 +1136,76 @@ export async function readScreenWith(deviceQuery, { useAx = true, useOcr = true,
1068
1136
  return { device, entry, points: { width: geo.pointWidth, height: geo.pointHeight } };
1069
1137
  }
1070
1138
 
1071
- export async function locate(
1139
+ /**
1140
+ * A target that answers the query but sits outside the viewport.
1141
+ *
1142
+ * The off-screen filter above is right — an element in the tree below the fold
1143
+ * is untappable in fact — but "not on this screen" was the wrong way to say so.
1144
+ * Reported: a `waitFor` spent its full 15s timeout and stopped the flow while
1145
+ * the control sat one scroll down. Waiting cannot fix that and scrolling can.
1146
+ */
1147
+ export function offScreenMatch(targets, query, points) {
1148
+ // Both axes: a horizontal row puts elements past the right edge, and checking
1149
+ // only `y` reported them as visible.
1150
+ const off = (targets ?? []).filter((t) => t.label && regions.offViewport(t, points));
1151
+ if (!off.length) return null;
1152
+ const hit = matching.resolve(off, query);
1153
+ if (hit.status === 'ok') return hit.target;
1154
+ // Ambiguous off-screen is still an answer to "why is it not here".
1155
+ return hit.status === 'ambiguous' && hit.alternatives?.length ? hit.alternatives[0] : null;
1156
+ }
1157
+
1158
+ /**
1159
+ * Which sensors a read asks for by default.
1160
+ *
1161
+ * `full` is the default and what CLAUDE.md fixes: accessibility and OCR fused
1162
+ * into one element list. `ax-first` asks for the tree alone — 85 ms against
1163
+ * 142 ms warm — and pays for OCR only when the cheap read could not answer the
1164
+ * question.
1165
+ *
1166
+ * The escalation is the whole point, and it is what makes this safe to try. A
1167
+ * mode that just dropped OCR would lose every OCR-only element, which is
1168
+ * precisely how the map cut lost discovery: nothing became untappable, but the
1169
+ * agent could no longer see what was there. Here a resolve failure — the one
1170
+ * signal that says "the cheap sensor was not enough" — triggers a full read
1171
+ * before anyone is told the target is absent. Wrong guesses cost a second read;
1172
+ * they cannot cost a wrong answer.
1173
+ */
1174
+ export function sensorMode(options) {
1175
+ // Per call first, then the environment. The environment is fixed when a
1176
+ // process starts, and an MCP server is one long-lived process — so a tester
1177
+ // asked to compare two sensor modes in one session could not do it, which is
1178
+ // exactly what happened: round 6's A was run and B and C could not be. Their
1179
+ // own suggestion was three server entries with three env blocks, and it is
1180
+ // the worse fix: three servers on one device means three writers, against the
1181
+ // one-writer-per-device rule, and it makes an A/B a configuration change
1182
+ // rather than an argument.
1183
+ const asked = options?.sensor;
1184
+ const raw = String(asked ?? process.env.SIMFRAME_SENSOR ?? '').trim().toLowerCase();
1185
+ return raw === 'ax-first' || raw === 'axfirst' ? 'ax-first' : 'full';
1186
+ }
1187
+
1188
+ export async function locate(deviceQuery, query, opts = {}) {
1189
+ if (sensorMode(opts.options) !== 'ax-first' || opts.useOcr === false || opts.escalated) {
1190
+ return locateWith(deviceQuery, query, opts);
1191
+ }
1192
+ const options = opts;
1193
+ try {
1194
+ return await locateWith(deviceQuery, query, { ...options, useOcr: false, escalated: true });
1195
+ } catch (err) {
1196
+ // Only a perception failure earns the expensive retry. A refused selector or
1197
+ // an ambiguity between two things the tree *did* see is not going to be
1198
+ // settled by reading more text.
1199
+ const why = metrics.escalationOf(err);
1200
+ if (why?.reason !== 'unknown_screen' && why?.reason !== 'ambiguous_intent') throw err;
1201
+ return locateWith(deviceQuery, query, { ...options, useOcr: true, refresh: true, escalated: true });
1202
+ }
1203
+ }
1204
+
1205
+ async function locateWith(
1072
1206
  deviceQuery,
1073
1207
  query,
1074
- { index, refresh = false, useAx = true, useOcr = true, settleMs = MEMORY_SETTLE_MS, options } = {},
1208
+ { index, refresh = false, useAx = true, useOcr = true, settleMs = MEMORY_SETTLE_MS, options, escalated } = {},
1075
1209
  ) {
1076
1210
  const { device, state: firstState } = await ensureDaemon(deviceQuery, options);
1077
1211
  const udid = device.udid;
@@ -1094,11 +1228,76 @@ export async function locate(
1094
1228
  // be checked against structural identity without paying for a perception
1095
1229
  // pass — which is the whole reason a ref exists.
1096
1230
  const near = screenmap.recallNearest(udid, firstState.layoutHash);
1097
- const hit = refs.resolveRef(udid, selector.ref, {
1098
- layoutHash: firstState.layoutHash,
1099
- structuralHash: near?.entry?.structuralHash ?? null,
1100
- screenKnown: Boolean(near),
1101
- });
1231
+ let hit;
1232
+ try {
1233
+ hit = resolveRefHere();
1234
+ } catch (err) {
1235
+ // A stale ref carries the label it was numbered against, so it need not
1236
+ // cost the rest of the batch. Reported: `#19 was numbered on a different
1237
+ // screen (61b835b7 → 6f34c006)` because dashboard cards finished loading
1238
+ // and shifted the layout — the same screen, a new hash — and that one
1239
+ // refusal aborted the three remaining steps.
1240
+ //
1241
+ // It re-resolves by label rather than by coordinate, and it *says so*.
1242
+ // The number is not honoured; the caller's own words are, which is what
1243
+ // they would have written instead. Resolving the old coordinates would be
1244
+ // the dangerous version of this, and is not what happens.
1245
+ // Only layout drift is recoverable. `identity` means the numbers were
1246
+ // drawn somewhere else, and the label they stood for appearing here is
1247
+ // coincidence rather than evidence — measured: `#1` numbered "Reminders"
1248
+ // in Reminders re-resolved in Contacts onto the status-bar back-to-app
1249
+ // breadcrumb "• Reminders", and reported ok.
1250
+ if (!err.staleRef || !err.staleLabel || err.staleKind !== 'drift') throw err;
1251
+ let again;
1252
+ try {
1253
+ again = await locateWith(deviceQuery, err.staleLabel, {
1254
+ index, refresh: true, useAx, useOcr, settleMs, options, escalated,
1255
+ });
1256
+ } catch (second) {
1257
+ // The number was not honoured and the label it stood for is not here
1258
+ // either. That is still a refusal to tap stale coordinates, but by the
1259
+ // time it surfaces it looks like an ordinary "not on this screen" and a
1260
+ // caller cannot tell the two apart. Carrying the marker across says
1261
+ // which question was actually asked.
1262
+ second.staleRef = true;
1263
+ second.staleLabel = err.staleLabel;
1264
+ throw second;
1265
+ }
1266
+ // A recovery is held to a higher bar than the lookup it stands in for,
1267
+ // and to one the caller never has to think about.
1268
+ //
1269
+ // `MINIMUM_SCORE` (0.45) is the bar for a label the caller actually
1270
+ // wrote. This label is one *we substituted on their behalf* after
1271
+ // refusing their `#n`, so a weak match here is not "close enough" — it is
1272
+ // us choosing a target nobody named. The reported wrong answer scored
1273
+ // **0.64** and cleared the ordinary floor comfortably.
1274
+ //
1275
+ // 0.8 is structural rather than fitted to that incident: a fuzzy name
1276
+ // match returns `similarity * 0.72` and a prefix match is scaled by its
1277
+ // coverage, so neither reaches 0.8 on the name alone. Only a near-exact
1278
+ // name does. And the region check needs no tuned number at all — if the
1279
+ // map would not offer this target, a recovery may not silently pick it.
1280
+ const region = again.target?.region ?? 'content';
1281
+ const weak = Number.isFinite(again.score) && again.score < RELABEL_MIN_SCORE;
1282
+ if (weak || !regions.offerable(region)) {
1283
+ err.message = `#${selector.ref} cannot be trusted here, and "${err.staleLabel}" was not`
1284
+ + ' safely re-findable either:'
1285
+ + (weak ? ` the best match scored ${again.score.toFixed(2)}, below the ${RELABEL_MIN_SCORE} a`
1286
+ + ' relabel needs (a number you did not ask for may not become a tap on a guess).' : '')
1287
+ + (!regions.offerable(region) ? ` the best match sits in the ${region}, which sim_ui does not`
1288
+ + ' offer as something to act on.' : '')
1289
+ + ' Read the screen again (sim_ui) and name the target.';
1290
+ err.staleRef = true;
1291
+ err.staleLabel = err.staleLabel;
1292
+ err.relabelRefused = { score: again.score ?? null, region };
1293
+ throw err;
1294
+ }
1295
+ return {
1296
+ ...again,
1297
+ from: 'ref-relabelled',
1298
+ relabelled: { ref: selector.ref, label: err.staleLabel, why: err.message },
1299
+ };
1300
+ }
1102
1301
  return {
1103
1302
  device,
1104
1303
  state: firstState,
@@ -1107,6 +1306,23 @@ export async function locate(
1107
1306
  distance: 0,
1108
1307
  settled: true,
1109
1308
  };
1309
+
1310
+ function resolveRefHere() {
1311
+ return refs.resolveRef(udid, selector.ref, {
1312
+ layoutHash: firstState.layoutHash,
1313
+ structuralHash: near?.entry?.structuralHash ?? null,
1314
+ // How far that recall reached. `recallNearest` is deliberately tolerant —
1315
+ // a list with new rows is still the same screen — so at any distance
1316
+ // above zero it is a *guess* about which screen this is, and a guess must
1317
+ // not be the sole grounds for refusing a ref. Reported from the field: a
1318
+ // refusal reading `#5 was numbered on a different screen
1319
+ // (0f7b9e3e → 48e5c92d)` where both calls' headers printed the identical
1320
+ // screen, because the map named the screen from a tolerant recall while
1321
+ // refs treated that same recall as exact.
1322
+ structuralDistance: near?.distance ?? null,
1323
+ screenKnown: Boolean(near),
1324
+ });
1325
+ }
1110
1326
  }
1111
1327
  if (selector.exact) query = selector.label;
1112
1328
  // Key memory off a settled frame, never off whichever frame happened to be
@@ -1167,7 +1383,7 @@ export async function locate(
1167
1383
  // ambiguous_intent when the screen was one we thought we knew. A
1168
1384
  // waiting caller needs the difference: more time cannot make a thing
1169
1385
  // unique, and it can make an absent thing arrive.
1170
- { candidates: outcome.alternatives, ambiguous: true },
1386
+ { candidates: outcome.alternatives, ambiguous: true, intent: query },
1171
1387
  );
1172
1388
  }
1173
1389
  if (outcome.status === 'ok') {
@@ -1181,8 +1397,25 @@ export async function locate(
1181
1397
  // here undoes every guard above — it has no off-screen filter and no
1182
1398
  // coverage weighting, and it is what returned a scrolled-away list row for
1183
1399
  // "back". "Not found" is the correct answer.
1184
- const visible = entry.targets.filter((t) => t.label && t.y >= 0 && t.y <= points.height);
1400
+ const visible = entry.targets.filter((t) => t.label && !regions.offViewport(t, points));
1185
1401
  const sample = visible.slice(0, 12).map((t) => t.label.slice(0, 24)).join(', ');
1402
+ // "Not on this screen" and "not in view" are different answers, and giving
1403
+ // the first for the second cost a reported 15 seconds: a `waitFor REVIEW`
1404
+ // burned its whole timeout while REVIEW sat one scroll below the fold, and
1405
+ // then stopped the flow. Waiting cannot bring a thing into view, and
1406
+ // scrolling can — so the difference is the whole of what to do next.
1407
+ const offScreen = offScreenMatch(entry.targets, query, points);
1408
+ if (offScreen) {
1409
+ throw metrics.tag(
1410
+ new Error(
1411
+ `"${query}" is in the tree but not in view — it is at y=${Math.round(offScreen.y)}`
1412
+ + ` on a ${Math.round(points.height)}pt screen. Scroll to it (sim_scroll_to) rather than waiting;`
1413
+ + ' waiting cannot bring it into view.',
1414
+ ),
1415
+ from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
1416
+ { candidates: visible.slice(0, 8), intent: query },
1417
+ );
1418
+ }
1186
1419
  // Which escalation this is depends on whether the screen was recognised.
1187
1420
  // Screen memory had nothing for it (`from` is one of the built values) and
1188
1421
  // the target is missing: that is not knowing the screen. On a screen
@@ -1190,7 +1423,7 @@ export async function locate(
1190
1423
  throw metrics.tag(
1191
1424
  new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
1192
1425
  from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
1193
- { candidates: visible.slice(0, 8) },
1426
+ { candidates: visible.slice(0, 8), intent: query },
1194
1427
  );
1195
1428
  }
1196
1429
 
@@ -1202,7 +1435,7 @@ export async function locate(
1202
1435
  throw metrics.tag(
1203
1436
  new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
1204
1437
  from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
1205
- { candidates: visible.slice(0, 8) },
1438
+ { candidates: visible.slice(0, 8), intent: query },
1206
1439
  );
1207
1440
  }
1208
1441
  return { device, state: current, entry, target, from, distance, settled, screens: screenmap.stats(udid).screens };
@@ -1315,6 +1548,11 @@ export async function screenIdentity(deviceQuery, { options, confirmNovel = true
1315
1548
  keyboard: Boolean(entry.keyboard),
1316
1549
  layoutHash: current.layoutHash,
1317
1550
  settled,
1551
+ // Settled and incomplete are different states and used to render
1552
+ // identically. A screen awaiting a network call is perfectly still; a
1553
+ // person sees a spinner and knows to wait. The classifier already says
1554
+ // so, and nothing above this line was asking.
1555
+ loading: current.transition?.kind === 'loading',
1318
1556
  // Carried out so callers that want the elements as well as the identity
1319
1557
  // do not pay for a second perception pass to get them. The compact
1320
1558
  // screen map needs both, and reading twice was the whole cost of it.
package/src/input.js CHANGED
@@ -429,14 +429,77 @@ export async function typeKeys(udid, value) {
429
429
  await idb(['ui', 'text', '--udid', udid, String(value)]);
430
430
  }
431
431
 
432
- export async function pressKey(udid, keycode) {
432
+ /**
433
+ * Keyboard keys, by name.
434
+ *
435
+ * A peer was blocked outright for want of Return: half of mobile search fields
436
+ * submit on the keyboard return key, `button` covers only the hardware buttons,
437
+ * and `key` wanted a raw HID usage code that nobody should have to know. Typing
438
+ * "\n" as text is not a substitute — text goes through whatever keyboard layout
439
+ * iOS has active, and measured, it turned "Coke Display" into "Coke In Display".
440
+ *
441
+ * These are HID keyboard usage codes, which name a key *position* and are never
442
+ * translated by a layout. That property is the whole reason this path exists on
443
+ * a device whose own doctor warns that two extra layouts are installed.
444
+ */
445
+ export const KEYS = {
446
+ return: 40, enter: 40, escape: 41, esc: 41, backspace: 42, delete: 42,
447
+ tab: 43, space: 44, up: 82, down: 81, left: 80, right: 79,
448
+ a: 4,
449
+ };
450
+
451
+ /** Modifier usage codes, held while another key is pressed. */
452
+ export const MODIFIERS = { control: 224, shift: 225, alt: 226, option: 226, command: 227, cmd: 227, gui: 227 };
453
+
454
+ /** The usage code for a name, a number, or null when it is neither. */
455
+ export function keyUsage(key) {
456
+ if (Number.isFinite(Number(key))) return Number(key);
457
+ const name = String(key ?? '').trim().toLowerCase();
458
+ return Object.hasOwn(KEYS, name) ? KEYS[name] : null;
459
+ }
460
+
461
+ export async function pressKey(udid, keycode, { modifiers = [] } = {}) {
433
462
  await ensureFreshSession(udid);
463
+ const usage = keyUsage(keycode);
464
+ const held = modifiers
465
+ .map((m) => (Number.isFinite(Number(m)) ? Number(m) : MODIFIERS[String(m).trim().toLowerCase()]))
466
+ .filter((m) => Number.isFinite(m));
467
+ if (usage == null) {
468
+ throw new Error(`unknown key ${JSON.stringify(String(keycode))} — known names: ${Object.keys(KEYS).join(', ')}`
469
+ + ', or a HID usage code');
470
+ }
434
471
  const own = inputDriverFor(udid);
435
472
  if (own) {
436
- await own.key(udid, keycode);
473
+ await own.key(udid, usage, held);
474
+ return;
475
+ }
476
+ // The daemon owns the keyboard usage path on iOS. It was implemented in the
477
+ // HID layer and never exposed as a verb, so this fell through to idb — which
478
+ // is absent on a machine using the daemon, and so there was no way to press a
479
+ // keyboard key at all.
480
+ if (control.available(udid)) {
481
+ await control.key(udid, usage, held);
437
482
  return;
438
483
  }
439
- await idb(['ui', 'key', '--udid', udid, String(keycode)]);
484
+ if (held.length) throw new Error('modifier keys need the daemon; idb cannot hold one');
485
+ await idb(['ui', 'key', '--udid', udid, String(usage)]);
486
+ }
487
+
488
+ /**
489
+ * Empty the focused field.
490
+ *
491
+ * Command-A then Delete, over HID. There is no clear primitive anywhere —
492
+ * XCUITest, Appium and idb all lack one, and re-typing appends — so this is the
493
+ * standard answer rather than a trick of ours. It is layout-independent for the
494
+ * reason that matters on a device with Farsi and Armenian keyboards installed:
495
+ * a modifier and Delete are key *positions*, and so is the `a` in Command-A, so
496
+ * none of the three is translated by the active layout.
497
+ *
498
+ * It clears whatever has focus, which is why every caller focuses first.
499
+ */
500
+ export async function clearField(udid) {
501
+ await pressKey(udid, 'a', { modifiers: ['command'] });
502
+ await pressKey(udid, 'delete');
440
503
  }
441
504
 
442
505
  /**
@@ -0,0 +1,161 @@
1
+ /**
2
+ * A warm, line-oriented local helper process.
3
+ *
4
+ * Extracted rather than duplicated, because writing this twice would mean
5
+ * risking the same two bugs twice — and both were subtle enough to look like
6
+ * something else entirely.
7
+ *
8
+ * A timed-out request left its waiter in the queue, so every later answer went
9
+ * to the wrong asker and the run simply never finished; it read as the model
10
+ * being slow. And unreferencing the child's stdout unreferenced the pipe every
11
+ * request waits on, so the process exited silently in the middle of an await
12
+ * and printed nothing at all, returning 0.
13
+ *
14
+ * The helper is kept warm because the first answer in a process pays model load
15
+ * — measured at ~880ms against ~560ms for every answer after it.
16
+ */
17
+ import { spawn, execFile } from 'node:child_process';
18
+ import fs from 'node:fs';
19
+ import path from 'node:path';
20
+ import { promisify } from 'node:util';
21
+
22
+ const run = promisify(execFile);
23
+
24
+ /**
25
+ * Compile a Swift source once and reuse the binary.
26
+ *
27
+ * `swiftc`, not `xcrun swiftc`: nothing above the platform boundary may name a
28
+ * platform tool, and the boundary test catches it. A compiler is not a device
29
+ * tool, which is the precedent `src/ocr.js` set.
30
+ */
31
+ export function compiler({ source, binary, what }) {
32
+ let building = null;
33
+ return async function ensureBinary() {
34
+ if (building) return building;
35
+ building = (async () => {
36
+ try {
37
+ const src = fs.statSync(source).mtimeMs;
38
+ const bin = fs.existsSync(binary) ? fs.statSync(binary).mtimeMs : 0;
39
+ if (bin > src) return { available: true, binary };
40
+ } catch {
41
+ return { available: false, reason: `the ${what} source is missing from this install` };
42
+ }
43
+ try {
44
+ fs.mkdirSync(path.dirname(binary), { recursive: true });
45
+ await run('swiftc', ['-O', source, '-o', binary], { timeout: 180_000 });
46
+ return { available: true, binary };
47
+ } catch (err) {
48
+ building = null; // let a later call retry once a toolchain is present
49
+ return {
50
+ available: false,
51
+ reason: err.code === 'ENOENT'
52
+ ? `swiftc is not installed, so the ${what} cannot be built (install Xcode command line tools)`
53
+ : `could not build the ${what}: ${String(err.message).split('\n')[0]}`,
54
+ };
55
+ }
56
+ })();
57
+ return building;
58
+ };
59
+ }
60
+
61
+ /**
62
+ * Open a helper and speak JSON lines to it.
63
+ *
64
+ * @returns {{ask: (o: object, ms: number) => Promise<object|null>, close: () => void, ok: boolean, reason?: string}}
65
+ */
66
+ export function lineServer({ ensureBinary, what }) {
67
+ let session = null;
68
+
69
+ async function open() {
70
+ if (session) return session;
71
+ const built = await ensureBinary();
72
+ if (!built.available) return { ok: false, reason: built.reason };
73
+ session = await new Promise((resolve) => {
74
+ const child = spawn(built.binary, [], { stdio: ['pipe', 'pipe', 'ignore'] });
75
+ // Deliberately NOT unref'd — see the note at the top of this file.
76
+ let buffer = '';
77
+ const waiters = [];
78
+ let settled = false;
79
+ const fail = (reason) => {
80
+ if (!settled) { settled = true; resolve({ ok: false, reason }); }
81
+ while (waiters.length) waiters.shift()(null);
82
+ };
83
+ child.on('error', (err) => fail(`the ${what} would not start: ${err.message}`));
84
+ child.on('exit', () => { session = null; fail(`the ${what} exited`); });
85
+ child.stdout.on('data', (chunk) => {
86
+ buffer += chunk;
87
+ let i = buffer.indexOf('\n');
88
+ while (i >= 0) {
89
+ const line = buffer.slice(0, i).trim();
90
+ buffer = buffer.slice(i + 1);
91
+ i = buffer.indexOf('\n');
92
+ if (!line) continue;
93
+ let msg;
94
+ try { msg = JSON.parse(line); } catch { continue; }
95
+ if (!settled) {
96
+ settled = true;
97
+ // The ready line may carry facts about the helper worth keeping —
98
+ // the model's context window, for one, which used to be a constant
99
+ // we repeated in comments. Passed through rather than parsed here,
100
+ // because this file knows about lines and not about models.
101
+ if (msg.ready) resolve({ ok: true, child, waiters, hello: msg });
102
+ else resolve({ ok: false, reason: msg.unavailable ?? `the ${what} did not become ready` });
103
+ continue;
104
+ }
105
+ const next = waiters.shift();
106
+ if (next) next(msg);
107
+ }
108
+ });
109
+ });
110
+ return session;
111
+ }
112
+
113
+ return {
114
+ async ask(question, timeoutMs = 3000) {
115
+ let live;
116
+ try {
117
+ live = await open();
118
+ } catch {
119
+ return null;
120
+ }
121
+ if (!live?.ok) return null;
122
+ return new Promise((resolve) => {
123
+ // A timed-out waiter is retired, not merely resolved. Leaving it queued
124
+ // sent the next answer to it instead of to the next asker, and every
125
+ // call after that was off by one.
126
+ let done = false;
127
+ const waiter = (msg) => {
128
+ if (done) return;
129
+ done = true;
130
+ clearTimeout(timer);
131
+ resolve(msg);
132
+ };
133
+ const timer = setTimeout(() => {
134
+ if (done) return;
135
+ done = true;
136
+ const i = live.waiters.indexOf(waiter);
137
+ if (i >= 0) live.waiters.splice(i, 1);
138
+ resolve(null);
139
+ }, timeoutMs);
140
+ live.waiters.push(waiter);
141
+ try {
142
+ live.child.stdin.write(`${JSON.stringify(question)}\n`);
143
+ } catch {
144
+ clearTimeout(timer);
145
+ done = true;
146
+ resolve(null);
147
+ }
148
+ });
149
+ },
150
+ async status() {
151
+ const live = await open();
152
+ return live?.ok
153
+ ? { ok: true, hello: live.hello ?? {} }
154
+ : { ok: false, reason: live?.reason ?? 'unavailable' };
155
+ },
156
+ close() {
157
+ try { session?.child?.kill(); } catch { /* already gone */ }
158
+ session = null;
159
+ },
160
+ };
161
+ }