simframe 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +181 -2
- package/data/vocabulary/en.json +148 -0
- package/native/ocr.swift +13 -1
- package/native/rank.swift +87 -0
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +72 -4
- package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +27 -0
- package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +4 -0
- package/native/simframed/Sources/SimframeCore/CaptureRecovery.swift +19 -0
- package/native/simframed/Sources/simframed/main.swift +33 -3
- package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +41 -0
- package/native/supervise.swift +216 -0
- package/package.json +4 -1
- package/scripts/check-package.mjs +22 -2
- package/scripts/check-private.mjs +9 -0
- package/scripts/ci-integration-local.sh +79 -0
- package/scripts/ci-memory.mjs +104 -20
- package/scripts/collect-rulings.mjs +312 -0
- package/scripts/eval-fingerprint.mjs +100 -23
- package/scripts/eval-perception.mjs +33 -0
- package/scripts/phase17-corpus.mjs +176 -0
- package/scripts/probe-network.mjs +118 -0
- package/scripts/soak-capture.mjs +72 -0
- package/skills/simframe/SKILL.md +257 -7
- package/src/actions.js +1791 -44
- package/src/cli.js +216 -14
- package/src/control.js +1 -0
- package/src/fingerprint.js +43 -1
- package/src/graph.js +136 -7
- package/src/index.js +252 -14
- package/src/input.js +66 -3
- package/src/localhelper.js +161 -0
- package/src/matching.js +64 -1
- package/src/mcp.js +333 -32
- package/src/metrics.js +148 -3
- package/src/ocr.js +18 -1
- package/src/planner.js +195 -0
- package/src/platform/android.js +25 -1
- package/src/platform/index.js +11 -1
- package/src/platform/ios.js +25 -1
- package/src/png.js +26 -0
- package/src/refs.js +51 -8
- package/src/regions.js +215 -1
- package/src/screenmap.js +89 -9
- package/src/supervisor.js +161 -0
- package/src/view.js +375 -11
- package/src/vocabulary.js +134 -0
- package/src/wrote.js +136 -0
package/src/index.js
CHANGED
|
@@ -5,6 +5,7 @@ import path from 'node:path';
|
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
6
6
|
import { DEFAULTS, STATE_VERSION } from './daemon.js';
|
|
7
7
|
import * as engine from './engine.js';
|
|
8
|
+
import * as regions from './regions.js';
|
|
8
9
|
import { decodePng, encodePng, scaleBitmap } from './png.js';
|
|
9
10
|
import {
|
|
10
11
|
REGION_COLS,
|
|
@@ -302,6 +303,18 @@ export const STALE_FRAME_MS = 2500;
|
|
|
302
303
|
*/
|
|
303
304
|
export const MEMORY_SETTLE_MS = 250;
|
|
304
305
|
|
|
306
|
+
/**
|
|
307
|
+
* How well a relabelled ref must match before it is acted on.
|
|
308
|
+
*
|
|
309
|
+
* Above `matching.MINIMUM_SCORE` (0.45) on purpose: that floor is for a label
|
|
310
|
+
* the caller wrote, and this is a label simframe substituted after refusing
|
|
311
|
+
* their `#n`. A fuzzy name match returns `similarity * 0.72` and a prefix match
|
|
312
|
+
* is scaled by its coverage, so neither can reach 0.8 on the name alone — which
|
|
313
|
+
* makes this "the label matched nearly exactly", not a tuned constant. The
|
|
314
|
+
* recovery that made this necessary scored 0.64.
|
|
315
|
+
*/
|
|
316
|
+
export const RELABEL_MIN_SCORE = 0.8;
|
|
317
|
+
|
|
305
318
|
/** Below this a "change" is a clock digit or a caret, not a new screen. */
|
|
306
319
|
export const MINOR_CHANGE = 0.004;
|
|
307
320
|
export const MAJOR_CHANGE = 0.03;
|
|
@@ -536,9 +549,31 @@ export async function getFrame(deviceQuery, { detail = 'normal', options } = {})
|
|
|
536
549
|
let scaledOnRead = false;
|
|
537
550
|
if (maxDim === 0) {
|
|
538
551
|
file = state.fullFile;
|
|
539
|
-
} else if (maxDim > nativeMax + 8
|
|
552
|
+
} else if (maxDim > nativeMax + 8) {
|
|
553
|
+
// Asking for more detail than the ring frame holds, so it has to come from
|
|
554
|
+
// a full-resolution frame. The `fs.existsSync(state.fullFile)` this used to
|
|
555
|
+
// require is exactly the condition that fails routinely — retention thins
|
|
556
|
+
// full frames aggressively, and `state.fullFile` names one that is often
|
|
557
|
+
// already gone. When it did, this fell straight through to `latest.png` and
|
|
558
|
+
// returned a 322x700 image while still calling itself detail "high".
|
|
559
|
+
//
|
|
560
|
+
// That was not a small inaccuracy. Every single `sim_look` header in a
|
|
561
|
+
// two-agent field round read `322x700` on a 402x874pt device, so "high
|
|
562
|
+
// (1024px, readable small text)" was interpolating a sub-1x source the
|
|
563
|
+
// whole time. `fullFrameFor` has always known how to find or take a full
|
|
564
|
+
// frame — state named full/9660.png while the directory held eight frames
|
|
565
|
+
// at 1206x2622 — and this path simply never asked it.
|
|
566
|
+
//
|
|
567
|
+
// What this does *not* explain, though it is tempting: the OCR corruption
|
|
568
|
+
// in that round's log (`saaiad@example.com`, `suomit order`). OCR has
|
|
569
|
+
// always gone through `fullFrameFor`, and the daemon's own text recognition
|
|
570
|
+
// reads the live surface at native resolution, so neither was ever looking
|
|
571
|
+
// at the small frame. Those errors are a desktop-width page rendering its
|
|
572
|
+
// labels at a few pixels tall. Fixing what the caller *sees* does not fix
|
|
573
|
+
// what OCR reads, and saying so here keeps the next reader from assuming it did.
|
|
574
|
+
const source = await fullFrameFor(device.udid, state);
|
|
540
575
|
const out = path.join(p.dir, `read-${maxDim}.png`);
|
|
541
|
-
await resize(
|
|
576
|
+
await resize(source, out, maxDim);
|
|
542
577
|
file = out;
|
|
543
578
|
scaledOnRead = true;
|
|
544
579
|
} else if (maxDim < nativeMax - 8) {
|
|
@@ -550,17 +585,50 @@ export async function getFrame(deviceQuery, { detail = 'normal', options } = {})
|
|
|
550
585
|
|
|
551
586
|
const png = fs.readFileSync(file);
|
|
552
587
|
const bmp = pngSize(png);
|
|
588
|
+
// The age must describe the *bytes*, not the state that dates them.
|
|
589
|
+
//
|
|
590
|
+
// These are two different files. `state.json` is written when a frame is
|
|
591
|
+
// recorded; the image is a separate write. A freshly started daemon publishes
|
|
592
|
+
// fresh state while `latest.png` is still the previous session's — which is
|
|
593
|
+
// when a caller's first look of a session happens. Reported, and it is the
|
|
594
|
+
// worst failure this tool can have: a header reading `frame #714 · 84ms old ·
|
|
595
|
+
// still for 17173ms` above an image whose status-bar clock said 6:11 when the
|
|
596
|
+
// real time was 11:10. `sim_ui` was right in the same session because the
|
|
597
|
+
// accessibility tree is read live and in-process; only the image comes from a
|
|
598
|
+
// file, so only the image can be hours stale while the header says otherwise.
|
|
599
|
+
//
|
|
600
|
+
// Taking the *larger* of the two ages cannot overstate freshness. It can
|
|
601
|
+
// overstate staleness by however long the two writes are apart, which is 6ms
|
|
602
|
+
// measured on a healthy daemon — the right direction to be wrong in.
|
|
603
|
+
let fileAgeMs = null;
|
|
604
|
+
try { fileAgeMs = Date.now() - fs.statSync(file).mtimeMs; } catch { /* stat is advisory */ }
|
|
605
|
+
const stateAgeMs = Date.now() - state.capturedAt;
|
|
606
|
+
const ageMs = Number.isFinite(fileAgeMs) ? Math.max(stateAgeMs, fileAgeMs) : stateAgeMs;
|
|
553
607
|
return {
|
|
554
608
|
device,
|
|
555
609
|
state,
|
|
556
610
|
png,
|
|
557
611
|
width: bmp.width,
|
|
558
612
|
height: bmp.height,
|
|
559
|
-
ageMs
|
|
613
|
+
ageMs,
|
|
614
|
+
// Said out loud when the image is materially older than the state, because
|
|
615
|
+
// "this picture is not of the screen the rest of this response describes" is
|
|
616
|
+
// not something a caller can work out for themselves.
|
|
617
|
+
frameBehindMs: Number.isFinite(fileAgeMs) && fileAgeMs - stateAgeMs > FRAME_BEHIND_MS
|
|
618
|
+
? Math.round(fileAgeMs - stateAgeMs)
|
|
619
|
+
: null,
|
|
560
620
|
scaledOnRead,
|
|
561
621
|
};
|
|
562
622
|
}
|
|
563
623
|
|
|
624
|
+
/**
|
|
625
|
+
* How far the image may lag the state before it is worth saying so.
|
|
626
|
+
*
|
|
627
|
+
* Measured on a healthy daemon, the two writes land 6ms apart. A second is far
|
|
628
|
+
* outside that and far inside the hours-stale case this exists to catch.
|
|
629
|
+
*/
|
|
630
|
+
const FRAME_BEHIND_MS = 1000;
|
|
631
|
+
|
|
564
632
|
function pngSize(png) {
|
|
565
633
|
return { width: png.readUInt32BE(16), height: png.readUInt32BE(20) };
|
|
566
634
|
}
|
|
@@ -1068,10 +1136,76 @@ export async function readScreenWith(deviceQuery, { useAx = true, useOcr = true,
|
|
|
1068
1136
|
return { device, entry, points: { width: geo.pointWidth, height: geo.pointHeight } };
|
|
1069
1137
|
}
|
|
1070
1138
|
|
|
1071
|
-
|
|
1139
|
+
/**
|
|
1140
|
+
* A target that answers the query but sits outside the viewport.
|
|
1141
|
+
*
|
|
1142
|
+
* The off-screen filter above is right — an element in the tree below the fold
|
|
1143
|
+
* is untappable in fact — but "not on this screen" was the wrong way to say so.
|
|
1144
|
+
* Reported: a `waitFor` spent its full 15s timeout and stopped the flow while
|
|
1145
|
+
* the control sat one scroll down. Waiting cannot fix that and scrolling can.
|
|
1146
|
+
*/
|
|
1147
|
+
export function offScreenMatch(targets, query, points) {
|
|
1148
|
+
// Both axes: a horizontal row puts elements past the right edge, and checking
|
|
1149
|
+
// only `y` reported them as visible.
|
|
1150
|
+
const off = (targets ?? []).filter((t) => t.label && regions.offViewport(t, points));
|
|
1151
|
+
if (!off.length) return null;
|
|
1152
|
+
const hit = matching.resolve(off, query);
|
|
1153
|
+
if (hit.status === 'ok') return hit.target;
|
|
1154
|
+
// Ambiguous off-screen is still an answer to "why is it not here".
|
|
1155
|
+
return hit.status === 'ambiguous' && hit.alternatives?.length ? hit.alternatives[0] : null;
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
/**
|
|
1159
|
+
* Which sensors a read asks for by default.
|
|
1160
|
+
*
|
|
1161
|
+
* `full` is the default and what CLAUDE.md fixes: accessibility and OCR fused
|
|
1162
|
+
* into one element list. `ax-first` asks for the tree alone — 85 ms against
|
|
1163
|
+
* 142 ms warm — and pays for OCR only when the cheap read could not answer the
|
|
1164
|
+
* question.
|
|
1165
|
+
*
|
|
1166
|
+
* The escalation is the whole point, and it is what makes this safe to try. A
|
|
1167
|
+
* mode that just dropped OCR would lose every OCR-only element, which is
|
|
1168
|
+
* precisely how the map cut lost discovery: nothing became untappable, but the
|
|
1169
|
+
* agent could no longer see what was there. Here a resolve failure — the one
|
|
1170
|
+
* signal that says "the cheap sensor was not enough" — triggers a full read
|
|
1171
|
+
* before anyone is told the target is absent. Wrong guesses cost a second read;
|
|
1172
|
+
* they cannot cost a wrong answer.
|
|
1173
|
+
*/
|
|
1174
|
+
export function sensorMode(options) {
|
|
1175
|
+
// Per call first, then the environment. The environment is fixed when a
|
|
1176
|
+
// process starts, and an MCP server is one long-lived process — so a tester
|
|
1177
|
+
// asked to compare two sensor modes in one session could not do it, which is
|
|
1178
|
+
// exactly what happened: round 6's A was run and B and C could not be. Their
|
|
1179
|
+
// own suggestion was three server entries with three env blocks, and it is
|
|
1180
|
+
// the worse fix: three servers on one device means three writers, against the
|
|
1181
|
+
// one-writer-per-device rule, and it makes an A/B a configuration change
|
|
1182
|
+
// rather than an argument.
|
|
1183
|
+
const asked = options?.sensor;
|
|
1184
|
+
const raw = String(asked ?? process.env.SIMFRAME_SENSOR ?? '').trim().toLowerCase();
|
|
1185
|
+
return raw === 'ax-first' || raw === 'axfirst' ? 'ax-first' : 'full';
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
export async function locate(deviceQuery, query, opts = {}) {
|
|
1189
|
+
if (sensorMode(opts.options) !== 'ax-first' || opts.useOcr === false || opts.escalated) {
|
|
1190
|
+
return locateWith(deviceQuery, query, opts);
|
|
1191
|
+
}
|
|
1192
|
+
const options = opts;
|
|
1193
|
+
try {
|
|
1194
|
+
return await locateWith(deviceQuery, query, { ...options, useOcr: false, escalated: true });
|
|
1195
|
+
} catch (err) {
|
|
1196
|
+
// Only a perception failure earns the expensive retry. A refused selector or
|
|
1197
|
+
// an ambiguity between two things the tree *did* see is not going to be
|
|
1198
|
+
// settled by reading more text.
|
|
1199
|
+
const why = metrics.escalationOf(err);
|
|
1200
|
+
if (why?.reason !== 'unknown_screen' && why?.reason !== 'ambiguous_intent') throw err;
|
|
1201
|
+
return locateWith(deviceQuery, query, { ...options, useOcr: true, refresh: true, escalated: true });
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
async function locateWith(
|
|
1072
1206
|
deviceQuery,
|
|
1073
1207
|
query,
|
|
1074
|
-
{ index, refresh = false, useAx = true, useOcr = true, settleMs = MEMORY_SETTLE_MS, options } = {},
|
|
1208
|
+
{ index, refresh = false, useAx = true, useOcr = true, settleMs = MEMORY_SETTLE_MS, options, escalated } = {},
|
|
1075
1209
|
) {
|
|
1076
1210
|
const { device, state: firstState } = await ensureDaemon(deviceQuery, options);
|
|
1077
1211
|
const udid = device.udid;
|
|
@@ -1094,11 +1228,76 @@ export async function locate(
|
|
|
1094
1228
|
// be checked against structural identity without paying for a perception
|
|
1095
1229
|
// pass — which is the whole reason a ref exists.
|
|
1096
1230
|
const near = screenmap.recallNearest(udid, firstState.layoutHash);
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1231
|
+
let hit;
|
|
1232
|
+
try {
|
|
1233
|
+
hit = resolveRefHere();
|
|
1234
|
+
} catch (err) {
|
|
1235
|
+
// A stale ref carries the label it was numbered against, so it need not
|
|
1236
|
+
// cost the rest of the batch. Reported: `#19 was numbered on a different
|
|
1237
|
+
// screen (61b835b7 → 6f34c006)` because dashboard cards finished loading
|
|
1238
|
+
// and shifted the layout — the same screen, a new hash — and that one
|
|
1239
|
+
// refusal aborted the three remaining steps.
|
|
1240
|
+
//
|
|
1241
|
+
// It re-resolves by label rather than by coordinate, and it *says so*.
|
|
1242
|
+
// The number is not honoured; the caller's own words are, which is what
|
|
1243
|
+
// they would have written instead. Resolving the old coordinates would be
|
|
1244
|
+
// the dangerous version of this, and is not what happens.
|
|
1245
|
+
// Only layout drift is recoverable. `identity` means the numbers were
|
|
1246
|
+
// drawn somewhere else, and the label they stood for appearing here is
|
|
1247
|
+
// coincidence rather than evidence — measured: `#1` numbered "Reminders"
|
|
1248
|
+
// in Reminders re-resolved in Contacts onto the status-bar back-to-app
|
|
1249
|
+
// breadcrumb "• Reminders", and reported ok.
|
|
1250
|
+
if (!err.staleRef || !err.staleLabel || err.staleKind !== 'drift') throw err;
|
|
1251
|
+
let again;
|
|
1252
|
+
try {
|
|
1253
|
+
again = await locateWith(deviceQuery, err.staleLabel, {
|
|
1254
|
+
index, refresh: true, useAx, useOcr, settleMs, options, escalated,
|
|
1255
|
+
});
|
|
1256
|
+
} catch (second) {
|
|
1257
|
+
// The number was not honoured and the label it stood for is not here
|
|
1258
|
+
// either. That is still a refusal to tap stale coordinates, but by the
|
|
1259
|
+
// time it surfaces it looks like an ordinary "not on this screen" and a
|
|
1260
|
+
// caller cannot tell the two apart. Carrying the marker across says
|
|
1261
|
+
// which question was actually asked.
|
|
1262
|
+
second.staleRef = true;
|
|
1263
|
+
second.staleLabel = err.staleLabel;
|
|
1264
|
+
throw second;
|
|
1265
|
+
}
|
|
1266
|
+
// A recovery is held to a higher bar than the lookup it stands in for,
|
|
1267
|
+
// and to one the caller never has to think about.
|
|
1268
|
+
//
|
|
1269
|
+
// `MINIMUM_SCORE` (0.45) is the bar for a label the caller actually
|
|
1270
|
+
// wrote. This label is one *we substituted on their behalf* after
|
|
1271
|
+
// refusing their `#n`, so a weak match here is not "close enough" — it is
|
|
1272
|
+
// us choosing a target nobody named. The reported wrong answer scored
|
|
1273
|
+
// **0.64** and cleared the ordinary floor comfortably.
|
|
1274
|
+
//
|
|
1275
|
+
// 0.8 is structural rather than fitted to that incident: a fuzzy name
|
|
1276
|
+
// match returns `similarity * 0.72` and a prefix match is scaled by its
|
|
1277
|
+
// coverage, so neither reaches 0.8 on the name alone. Only a near-exact
|
|
1278
|
+
// name does. And the region check needs no tuned number at all — if the
|
|
1279
|
+
// map would not offer this target, a recovery may not silently pick it.
|
|
1280
|
+
const region = again.target?.region ?? 'content';
|
|
1281
|
+
const weak = Number.isFinite(again.score) && again.score < RELABEL_MIN_SCORE;
|
|
1282
|
+
if (weak || !regions.offerable(region)) {
|
|
1283
|
+
err.message = `#${selector.ref} cannot be trusted here, and "${err.staleLabel}" was not`
|
|
1284
|
+
+ ' safely re-findable either:'
|
|
1285
|
+
+ (weak ? ` the best match scored ${again.score.toFixed(2)}, below the ${RELABEL_MIN_SCORE} a`
|
|
1286
|
+
+ ' relabel needs (a number you did not ask for may not become a tap on a guess).' : '')
|
|
1287
|
+
+ (!regions.offerable(region) ? ` the best match sits in the ${region}, which sim_ui does not`
|
|
1288
|
+
+ ' offer as something to act on.' : '')
|
|
1289
|
+
+ ' Read the screen again (sim_ui) and name the target.';
|
|
1290
|
+
err.staleRef = true;
|
|
1291
|
+
err.staleLabel = err.staleLabel;
|
|
1292
|
+
err.relabelRefused = { score: again.score ?? null, region };
|
|
1293
|
+
throw err;
|
|
1294
|
+
}
|
|
1295
|
+
return {
|
|
1296
|
+
...again,
|
|
1297
|
+
from: 'ref-relabelled',
|
|
1298
|
+
relabelled: { ref: selector.ref, label: err.staleLabel, why: err.message },
|
|
1299
|
+
};
|
|
1300
|
+
}
|
|
1102
1301
|
return {
|
|
1103
1302
|
device,
|
|
1104
1303
|
state: firstState,
|
|
@@ -1107,6 +1306,23 @@ export async function locate(
|
|
|
1107
1306
|
distance: 0,
|
|
1108
1307
|
settled: true,
|
|
1109
1308
|
};
|
|
1309
|
+
|
|
1310
|
+
function resolveRefHere() {
|
|
1311
|
+
return refs.resolveRef(udid, selector.ref, {
|
|
1312
|
+
layoutHash: firstState.layoutHash,
|
|
1313
|
+
structuralHash: near?.entry?.structuralHash ?? null,
|
|
1314
|
+
// How far that recall reached. `recallNearest` is deliberately tolerant —
|
|
1315
|
+
// a list with new rows is still the same screen — so at any distance
|
|
1316
|
+
// above zero it is a *guess* about which screen this is, and a guess must
|
|
1317
|
+
// not be the sole grounds for refusing a ref. Reported from the field: a
|
|
1318
|
+
// refusal reading `#5 was numbered on a different screen
|
|
1319
|
+
// (0f7b9e3e → 48e5c92d)` where both calls' headers printed the identical
|
|
1320
|
+
// screen, because the map named the screen from a tolerant recall while
|
|
1321
|
+
// refs treated that same recall as exact.
|
|
1322
|
+
structuralDistance: near?.distance ?? null,
|
|
1323
|
+
screenKnown: Boolean(near),
|
|
1324
|
+
});
|
|
1325
|
+
}
|
|
1110
1326
|
}
|
|
1111
1327
|
if (selector.exact) query = selector.label;
|
|
1112
1328
|
// Key memory off a settled frame, never off whichever frame happened to be
|
|
@@ -1167,7 +1383,7 @@ export async function locate(
|
|
|
1167
1383
|
// ambiguous_intent when the screen was one we thought we knew. A
|
|
1168
1384
|
// waiting caller needs the difference: more time cannot make a thing
|
|
1169
1385
|
// unique, and it can make an absent thing arrive.
|
|
1170
|
-
{ candidates: outcome.alternatives, ambiguous: true },
|
|
1386
|
+
{ candidates: outcome.alternatives, ambiguous: true, intent: query },
|
|
1171
1387
|
);
|
|
1172
1388
|
}
|
|
1173
1389
|
if (outcome.status === 'ok') {
|
|
@@ -1181,8 +1397,25 @@ export async function locate(
|
|
|
1181
1397
|
// here undoes every guard above — it has no off-screen filter and no
|
|
1182
1398
|
// coverage weighting, and it is what returned a scrolled-away list row for
|
|
1183
1399
|
// "back". "Not found" is the correct answer.
|
|
1184
|
-
const visible = entry.targets.filter((t) => t.label &&
|
|
1400
|
+
const visible = entry.targets.filter((t) => t.label && !regions.offViewport(t, points));
|
|
1185
1401
|
const sample = visible.slice(0, 12).map((t) => t.label.slice(0, 24)).join(', ');
|
|
1402
|
+
// "Not on this screen" and "not in view" are different answers, and giving
|
|
1403
|
+
// the first for the second cost a reported 15 seconds: a `waitFor REVIEW`
|
|
1404
|
+
// burned its whole timeout while REVIEW sat one scroll below the fold, and
|
|
1405
|
+
// then stopped the flow. Waiting cannot bring a thing into view, and
|
|
1406
|
+
// scrolling can — so the difference is the whole of what to do next.
|
|
1407
|
+
const offScreen = offScreenMatch(entry.targets, query, points);
|
|
1408
|
+
if (offScreen) {
|
|
1409
|
+
throw metrics.tag(
|
|
1410
|
+
new Error(
|
|
1411
|
+
`"${query}" is in the tree but not in view — it is at y=${Math.round(offScreen.y)}`
|
|
1412
|
+
+ ` on a ${Math.round(points.height)}pt screen. Scroll to it (sim_scroll_to) rather than waiting;`
|
|
1413
|
+
+ ' waiting cannot bring it into view.',
|
|
1414
|
+
),
|
|
1415
|
+
from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
|
|
1416
|
+
{ candidates: visible.slice(0, 8), intent: query },
|
|
1417
|
+
);
|
|
1418
|
+
}
|
|
1186
1419
|
// Which escalation this is depends on whether the screen was recognised.
|
|
1187
1420
|
// Screen memory had nothing for it (`from` is one of the built values) and
|
|
1188
1421
|
// the target is missing: that is not knowing the screen. On a screen
|
|
@@ -1190,7 +1423,7 @@ export async function locate(
|
|
|
1190
1423
|
throw metrics.tag(
|
|
1191
1424
|
new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
|
|
1192
1425
|
from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
|
|
1193
|
-
{ candidates: visible.slice(0, 8) },
|
|
1426
|
+
{ candidates: visible.slice(0, 8), intent: query },
|
|
1194
1427
|
);
|
|
1195
1428
|
}
|
|
1196
1429
|
|
|
@@ -1202,7 +1435,7 @@ export async function locate(
|
|
|
1202
1435
|
throw metrics.tag(
|
|
1203
1436
|
new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
|
|
1204
1437
|
from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
|
|
1205
|
-
{ candidates: visible.slice(0, 8) },
|
|
1438
|
+
{ candidates: visible.slice(0, 8), intent: query },
|
|
1206
1439
|
);
|
|
1207
1440
|
}
|
|
1208
1441
|
return { device, state: current, entry, target, from, distance, settled, screens: screenmap.stats(udid).screens };
|
|
@@ -1315,6 +1548,11 @@ export async function screenIdentity(deviceQuery, { options, confirmNovel = true
|
|
|
1315
1548
|
keyboard: Boolean(entry.keyboard),
|
|
1316
1549
|
layoutHash: current.layoutHash,
|
|
1317
1550
|
settled,
|
|
1551
|
+
// Settled and incomplete are different states and used to render
|
|
1552
|
+
// identically. A screen awaiting a network call is perfectly still; a
|
|
1553
|
+
// person sees a spinner and knows to wait. The classifier already says
|
|
1554
|
+
// so, and nothing above this line was asking.
|
|
1555
|
+
loading: current.transition?.kind === 'loading',
|
|
1318
1556
|
// Carried out so callers that want the elements as well as the identity
|
|
1319
1557
|
// do not pay for a second perception pass to get them. The compact
|
|
1320
1558
|
// screen map needs both, and reading twice was the whole cost of it.
|
package/src/input.js
CHANGED
|
@@ -429,14 +429,77 @@ export async function typeKeys(udid, value) {
|
|
|
429
429
|
await idb(['ui', 'text', '--udid', udid, String(value)]);
|
|
430
430
|
}
|
|
431
431
|
|
|
432
|
-
|
|
432
|
+
/**
|
|
433
|
+
* Keyboard keys, by name.
|
|
434
|
+
*
|
|
435
|
+
* A peer was blocked outright for want of Return: half of mobile search fields
|
|
436
|
+
* submit on the keyboard return key, `button` covers only the hardware buttons,
|
|
437
|
+
* and `key` wanted a raw HID usage code that nobody should have to know. Typing
|
|
438
|
+
* "\n" as text is not a substitute — text goes through whatever keyboard layout
|
|
439
|
+
* iOS has active, and measured, it turned "Coke Display" into "Coke In Display".
|
|
440
|
+
*
|
|
441
|
+
* These are HID keyboard usage codes, which name a key *position* and are never
|
|
442
|
+
* translated by a layout. That property is the whole reason this path exists on
|
|
443
|
+
* a device whose own doctor warns that two extra layouts are installed.
|
|
444
|
+
*/
|
|
445
|
+
export const KEYS = {
|
|
446
|
+
return: 40, enter: 40, escape: 41, esc: 41, backspace: 42, delete: 42,
|
|
447
|
+
tab: 43, space: 44, up: 82, down: 81, left: 80, right: 79,
|
|
448
|
+
a: 4,
|
|
449
|
+
};
|
|
450
|
+
|
|
451
|
+
/** Modifier usage codes, held while another key is pressed. */
|
|
452
|
+
export const MODIFIERS = { control: 224, shift: 225, alt: 226, option: 226, command: 227, cmd: 227, gui: 227 };
|
|
453
|
+
|
|
454
|
+
/** The usage code for a name, a number, or null when it is neither. */
|
|
455
|
+
export function keyUsage(key) {
|
|
456
|
+
if (Number.isFinite(Number(key))) return Number(key);
|
|
457
|
+
const name = String(key ?? '').trim().toLowerCase();
|
|
458
|
+
return Object.hasOwn(KEYS, name) ? KEYS[name] : null;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
export async function pressKey(udid, keycode, { modifiers = [] } = {}) {
|
|
433
462
|
await ensureFreshSession(udid);
|
|
463
|
+
const usage = keyUsage(keycode);
|
|
464
|
+
const held = modifiers
|
|
465
|
+
.map((m) => (Number.isFinite(Number(m)) ? Number(m) : MODIFIERS[String(m).trim().toLowerCase()]))
|
|
466
|
+
.filter((m) => Number.isFinite(m));
|
|
467
|
+
if (usage == null) {
|
|
468
|
+
throw new Error(`unknown key ${JSON.stringify(String(keycode))} — known names: ${Object.keys(KEYS).join(', ')}`
|
|
469
|
+
+ ', or a HID usage code');
|
|
470
|
+
}
|
|
434
471
|
const own = inputDriverFor(udid);
|
|
435
472
|
if (own) {
|
|
436
|
-
await own.key(udid,
|
|
473
|
+
await own.key(udid, usage, held);
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
// The daemon owns the keyboard usage path on iOS. It was implemented in the
|
|
477
|
+
// HID layer and never exposed as a verb, so this fell through to idb — which
|
|
478
|
+
// is absent on a machine using the daemon, and so there was no way to press a
|
|
479
|
+
// keyboard key at all.
|
|
480
|
+
if (control.available(udid)) {
|
|
481
|
+
await control.key(udid, usage, held);
|
|
437
482
|
return;
|
|
438
483
|
}
|
|
439
|
-
|
|
484
|
+
if (held.length) throw new Error('modifier keys need the daemon; idb cannot hold one');
|
|
485
|
+
await idb(['ui', 'key', '--udid', udid, String(usage)]);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* Empty the focused field.
|
|
490
|
+
*
|
|
491
|
+
* Command-A then Delete, over HID. There is no clear primitive anywhere —
|
|
492
|
+
* XCUITest, Appium and idb all lack one, and re-typing appends — so this is the
|
|
493
|
+
* standard answer rather than a trick of ours. It is layout-independent for the
|
|
494
|
+
* reason that matters on a device with Farsi and Armenian keyboards installed:
|
|
495
|
+
* a modifier and Delete are key *positions*, and so is the `a` in Command-A, so
|
|
496
|
+
* none of the three is translated by the active layout.
|
|
497
|
+
*
|
|
498
|
+
* It clears whatever has focus, which is why every caller focuses first.
|
|
499
|
+
*/
|
|
500
|
+
export async function clearField(udid) {
|
|
501
|
+
await pressKey(udid, 'a', { modifiers: ['command'] });
|
|
502
|
+
await pressKey(udid, 'delete');
|
|
440
503
|
}
|
|
441
504
|
|
|
442
505
|
/**
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A warm, line-oriented local helper process.
|
|
3
|
+
*
|
|
4
|
+
* Extracted rather than duplicated, because writing this twice would mean
|
|
5
|
+
* risking the same two bugs twice — and both were subtle enough to look like
|
|
6
|
+
* something else entirely.
|
|
7
|
+
*
|
|
8
|
+
* A timed-out request left its waiter in the queue, so every later answer went
|
|
9
|
+
* to the wrong asker and the run simply never finished; it read as the model
|
|
10
|
+
* being slow. And unreferencing the child's stdout unreferenced the pipe every
|
|
11
|
+
* request waits on, so the process exited silently in the middle of an await
|
|
12
|
+
* and printed nothing at all, returning 0.
|
|
13
|
+
*
|
|
14
|
+
* The helper is kept warm because the first answer in a process pays model load
|
|
15
|
+
* — measured at ~880ms against ~560ms for every answer after it.
|
|
16
|
+
*/
|
|
17
|
+
import { spawn, execFile } from 'node:child_process';
|
|
18
|
+
import fs from 'node:fs';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
import { promisify } from 'node:util';
|
|
21
|
+
|
|
22
|
+
const run = promisify(execFile);
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Compile a Swift source once and reuse the binary.
|
|
26
|
+
*
|
|
27
|
+
* `swiftc`, not `xcrun swiftc`: nothing above the platform boundary may name a
|
|
28
|
+
* platform tool, and the boundary test catches it. A compiler is not a device
|
|
29
|
+
* tool, which is the precedent `src/ocr.js` set.
|
|
30
|
+
*/
|
|
31
|
+
export function compiler({ source, binary, what }) {
|
|
32
|
+
let building = null;
|
|
33
|
+
return async function ensureBinary() {
|
|
34
|
+
if (building) return building;
|
|
35
|
+
building = (async () => {
|
|
36
|
+
try {
|
|
37
|
+
const src = fs.statSync(source).mtimeMs;
|
|
38
|
+
const bin = fs.existsSync(binary) ? fs.statSync(binary).mtimeMs : 0;
|
|
39
|
+
if (bin > src) return { available: true, binary };
|
|
40
|
+
} catch {
|
|
41
|
+
return { available: false, reason: `the ${what} source is missing from this install` };
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
fs.mkdirSync(path.dirname(binary), { recursive: true });
|
|
45
|
+
await run('swiftc', ['-O', source, '-o', binary], { timeout: 180_000 });
|
|
46
|
+
return { available: true, binary };
|
|
47
|
+
} catch (err) {
|
|
48
|
+
building = null; // let a later call retry once a toolchain is present
|
|
49
|
+
return {
|
|
50
|
+
available: false,
|
|
51
|
+
reason: err.code === 'ENOENT'
|
|
52
|
+
? `swiftc is not installed, so the ${what} cannot be built (install Xcode command line tools)`
|
|
53
|
+
: `could not build the ${what}: ${String(err.message).split('\n')[0]}`,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
})();
|
|
57
|
+
return building;
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Open a helper and speak JSON lines to it.
|
|
63
|
+
*
|
|
64
|
+
* @returns {{ask: (o: object, ms: number) => Promise<object|null>, close: () => void, ok: boolean, reason?: string}}
|
|
65
|
+
*/
|
|
66
|
+
export function lineServer({ ensureBinary, what }) {
|
|
67
|
+
let session = null;
|
|
68
|
+
|
|
69
|
+
async function open() {
|
|
70
|
+
if (session) return session;
|
|
71
|
+
const built = await ensureBinary();
|
|
72
|
+
if (!built.available) return { ok: false, reason: built.reason };
|
|
73
|
+
session = await new Promise((resolve) => {
|
|
74
|
+
const child = spawn(built.binary, [], { stdio: ['pipe', 'pipe', 'ignore'] });
|
|
75
|
+
// Deliberately NOT unref'd — see the note at the top of this file.
|
|
76
|
+
let buffer = '';
|
|
77
|
+
const waiters = [];
|
|
78
|
+
let settled = false;
|
|
79
|
+
const fail = (reason) => {
|
|
80
|
+
if (!settled) { settled = true; resolve({ ok: false, reason }); }
|
|
81
|
+
while (waiters.length) waiters.shift()(null);
|
|
82
|
+
};
|
|
83
|
+
child.on('error', (err) => fail(`the ${what} would not start: ${err.message}`));
|
|
84
|
+
child.on('exit', () => { session = null; fail(`the ${what} exited`); });
|
|
85
|
+
child.stdout.on('data', (chunk) => {
|
|
86
|
+
buffer += chunk;
|
|
87
|
+
let i = buffer.indexOf('\n');
|
|
88
|
+
while (i >= 0) {
|
|
89
|
+
const line = buffer.slice(0, i).trim();
|
|
90
|
+
buffer = buffer.slice(i + 1);
|
|
91
|
+
i = buffer.indexOf('\n');
|
|
92
|
+
if (!line) continue;
|
|
93
|
+
let msg;
|
|
94
|
+
try { msg = JSON.parse(line); } catch { continue; }
|
|
95
|
+
if (!settled) {
|
|
96
|
+
settled = true;
|
|
97
|
+
// The ready line may carry facts about the helper worth keeping —
|
|
98
|
+
// the model's context window, for one, which used to be a constant
|
|
99
|
+
// we repeated in comments. Passed through rather than parsed here,
|
|
100
|
+
// because this file knows about lines and not about models.
|
|
101
|
+
if (msg.ready) resolve({ ok: true, child, waiters, hello: msg });
|
|
102
|
+
else resolve({ ok: false, reason: msg.unavailable ?? `the ${what} did not become ready` });
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
const next = waiters.shift();
|
|
106
|
+
if (next) next(msg);
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
return session;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
return {
|
|
114
|
+
async ask(question, timeoutMs = 3000) {
|
|
115
|
+
let live;
|
|
116
|
+
try {
|
|
117
|
+
live = await open();
|
|
118
|
+
} catch {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
if (!live?.ok) return null;
|
|
122
|
+
return new Promise((resolve) => {
|
|
123
|
+
// A timed-out waiter is retired, not merely resolved. Leaving it queued
|
|
124
|
+
// sent the next answer to it instead of to the next asker, and every
|
|
125
|
+
// call after that was off by one.
|
|
126
|
+
let done = false;
|
|
127
|
+
const waiter = (msg) => {
|
|
128
|
+
if (done) return;
|
|
129
|
+
done = true;
|
|
130
|
+
clearTimeout(timer);
|
|
131
|
+
resolve(msg);
|
|
132
|
+
};
|
|
133
|
+
const timer = setTimeout(() => {
|
|
134
|
+
if (done) return;
|
|
135
|
+
done = true;
|
|
136
|
+
const i = live.waiters.indexOf(waiter);
|
|
137
|
+
if (i >= 0) live.waiters.splice(i, 1);
|
|
138
|
+
resolve(null);
|
|
139
|
+
}, timeoutMs);
|
|
140
|
+
live.waiters.push(waiter);
|
|
141
|
+
try {
|
|
142
|
+
live.child.stdin.write(`${JSON.stringify(question)}\n`);
|
|
143
|
+
} catch {
|
|
144
|
+
clearTimeout(timer);
|
|
145
|
+
done = true;
|
|
146
|
+
resolve(null);
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
},
|
|
150
|
+
async status() {
|
|
151
|
+
const live = await open();
|
|
152
|
+
return live?.ok
|
|
153
|
+
? { ok: true, hello: live.hello ?? {} }
|
|
154
|
+
: { ok: false, reason: live?.reason ?? 'unavailable' };
|
|
155
|
+
},
|
|
156
|
+
close() {
|
|
157
|
+
try { session?.child?.kill(); } catch { /* already gone */ }
|
|
158
|
+
session = null;
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
}
|