simframe 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -5
- package/data/vocabulary/en.json +148 -0
- package/native/ocr.swift +13 -1
- package/native/rank.swift +87 -0
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +43 -3
- package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +27 -0
- package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +4 -0
- package/native/simframed/Sources/simframed/main.swift +13 -1
- package/native/supervise.swift +181 -0
- package/package.json +4 -1
- package/scripts/check-package.mjs +22 -2
- package/scripts/check-private.mjs +143 -0
- package/scripts/ci-memory.mjs +104 -20
- package/scripts/eval-perception.mjs +281 -0
- package/scripts/phase17-corpus.mjs +176 -0
- package/skills/simframe/SKILL.md +237 -5
- package/src/actions.js +1825 -38
- package/src/analyze.js +70 -0
- package/src/cli.js +214 -15
- package/src/control.js +1 -0
- package/src/fingerprint.js +19 -1
- package/src/graph.js +193 -11
- package/src/index.js +428 -16
- package/src/input.js +115 -8
- package/src/localhelper.js +155 -0
- package/src/matching.js +119 -3
- package/src/mcp.js +319 -27
- package/src/metrics.js +134 -8
- package/src/navigate.js +10 -7
- package/src/ocr.js +18 -1
- package/src/planner.js +195 -0
- package/src/platform/android.js +3 -2
- package/src/platform/ios.js +2 -1
- package/src/png.js +26 -0
- package/src/refs.js +51 -8
- package/src/regions.js +110 -1
- package/src/screenmap.js +109 -10
- package/src/supervisor.js +117 -0
- package/src/view.js +396 -7
- package/src/vocabulary.js +134 -0
- package/src/wrote.js +136 -0
package/src/analyze.js
CHANGED
|
@@ -35,6 +35,42 @@ export function signatureDiff(a, b) {
|
|
|
35
35
|
return sum / a.length / 255;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
/**
|
|
39
|
+
* A change small enough that the whole-screen mean cannot see it.
|
|
40
|
+
*
|
|
41
|
+
* `changed` in both daemons is `signatureDiff > 0.004`, a mean over a 4x8 grid
|
|
42
|
+
* of gray means. Measured on this device, an iOS switch flipping:
|
|
43
|
+
*
|
|
44
|
+
* mean diff 0.001348 — a third of the threshold, so: not a change
|
|
45
|
+
* max cell delta 0.043137 — one cell of thirty-two, in row 1
|
|
46
|
+
*
|
|
47
|
+
* So the entire class of small binary controls — switches, radio dots,
|
|
48
|
+
* checkboxes, segment highlights — changes nothing as far as the daemon is
|
|
49
|
+
* concerned, and a step that flips one reports `no-visible-change`, which is a
|
|
50
|
+
* verdict that escalates.
|
|
51
|
+
*
|
|
52
|
+
* The threshold sits in a measured gap rather than being chosen. Eighty seconds
|
|
53
|
+
* of a *static* screen gave a largest per-cell delta of 0.003922, in row 0,
|
|
54
|
+
* which is the status-bar clock ticking over — the only thing moving. So the
|
|
55
|
+
* separation is 0.0039 against 0.0431, eleven times, and 0.012 is three times
|
|
56
|
+
* the noise and three and a half times under the signal. No row is excluded:
|
|
57
|
+
* the clock does not reach the threshold, which is a better reason to ignore it
|
|
58
|
+
* than a structural exclusion that would also blind the nav bar.
|
|
59
|
+
*
|
|
60
|
+
* What this deliberately does **not** do is feed stillness. `stableForMs` stays
|
|
61
|
+
* on the mean, because a blinking text caret is a small localised change and a
|
|
62
|
+
* screen with a cursor in it would otherwise never settle. The two signals are
|
|
63
|
+
* independent by design: this one answers "did the action do anything", and the
|
|
64
|
+
* mean answers "has the screen finished moving".
|
|
65
|
+
*/
|
|
66
|
+
export const CELL_CHANGE = 0.012;
|
|
67
|
+
|
|
68
|
+
/** The largest single-region change between two signatures, 0-1. */
|
|
69
|
+
export function maxCellDelta(a, b) {
|
|
70
|
+
const deltas = regionDeltas(a, b);
|
|
71
|
+
return deltas.length ? Math.max(...deltas) : 0;
|
|
72
|
+
}
|
|
73
|
+
|
|
38
74
|
/** Per-region change fractions, so callers can tell a toast from a screen push. */
|
|
39
75
|
export function regionDeltas(a, b) {
|
|
40
76
|
if (!a || !b || a.length !== b.length) return a ? a.map(() => 1) : [];
|
|
@@ -69,6 +105,40 @@ export function signatureToHex(sig) {
|
|
|
69
105
|
return sig.map((v) => v.toString(16).padStart(2, '0')).join('');
|
|
70
106
|
}
|
|
71
107
|
|
|
108
|
+
/**
|
|
109
|
+
* Is this frame black — not dark, black?
|
|
110
|
+
*
|
|
111
|
+
* The capture wedge (docs/BENCHMARKS.md) leaves `simctl io screenshot`
|
|
112
|
+
* succeeding and returning 0 non-black pixels of 3,162,132, and simframe's own
|
|
113
|
+
* frames go the same way: the display pipeline stops rendering while
|
|
114
|
+
* everything about the capture path keeps reporting success. It self-recovers
|
|
115
|
+
* most times and a device restart cures the rest, so the useful thing is to
|
|
116
|
+
* notice early — before a settle spends its whole budget deciding a black
|
|
117
|
+
* screen is a calm one.
|
|
118
|
+
*
|
|
119
|
+
* The signature is already computed for every frame, so this costs 32 integer
|
|
120
|
+
* comparisons and no decode. A real screen does not come close: measured on
|
|
121
|
+
* this device's Settings root, the 32 bytes ran 191–245.
|
|
122
|
+
*
|
|
123
|
+
* The threshold is a level, not a fraction, and it is on the *maximum*: one
|
|
124
|
+
* cell with anything in it is enough to say the display is rendering. That
|
|
125
|
+
* matters because a dark-mode screen, a video, or a splash on black are all
|
|
126
|
+
* legitimately near-zero in most cells and this must not call them faults.
|
|
127
|
+
*
|
|
128
|
+
* And it says "the frames are black", never "the simulator is wedged". A
|
|
129
|
+
* screen can be black because the app drew black. What makes it a wedge is
|
|
130
|
+
* that it stays black while input is being delivered, and only the caller
|
|
131
|
+
* knows that.
|
|
132
|
+
*/
|
|
133
|
+
export const BLACK_LEVEL = 8;
|
|
134
|
+
|
|
135
|
+
export function isBlackFrame(sig, { level = BLACK_LEVEL } = {}) {
|
|
136
|
+
const bytes = typeof sig === 'string' ? hexToSignature(sig) : sig;
|
|
137
|
+
if (!bytes?.length) return false;
|
|
138
|
+
for (const b of bytes) if (b > level) return false;
|
|
139
|
+
return true;
|
|
140
|
+
}
|
|
141
|
+
|
|
72
142
|
export function hexToSignature(hex) {
|
|
73
143
|
const out = [];
|
|
74
144
|
for (let i = 0; i < hex.length; i += 2) out.push(parseInt(hex.slice(i, i + 2), 16));
|
package/src/cli.js
CHANGED
|
@@ -5,6 +5,7 @@ import path from 'node:path';
|
|
|
5
5
|
import { runDaemon, DEFAULTS } from './daemon.js';
|
|
6
6
|
import { bootedDevices, capabilitiesFor, listDevices, PLATFORMS, resolveDevice, screenshot, toolchainChecks } from './platform/index.js';
|
|
7
7
|
import * as actions from './actions.js';
|
|
8
|
+
import * as analyze from './analyze.js';
|
|
8
9
|
import * as api from './index.js';
|
|
9
10
|
import * as input from './input.js';
|
|
10
11
|
import * as baseline from './baseline.js';
|
|
@@ -20,6 +21,7 @@ const USAGE = `simframe — always-warm iOS Simulator frames
|
|
|
20
21
|
simframe start [device] start the capture loop in the background
|
|
21
22
|
simframe stop [device|--all] stop the capture loop
|
|
22
23
|
simframe status [device] show daemon and newest-frame status
|
|
24
|
+
simframe input reset rebuild the HID session (see doctor)
|
|
23
25
|
simframe frame [device] write the newest frame to a file
|
|
24
26
|
simframe state [device] print frame metadata and the change map
|
|
25
27
|
simframe mark [device] print the current frame hash, to use as --since
|
|
@@ -45,6 +47,10 @@ const USAGE = `simframe — always-warm iOS Simulator frames
|
|
|
45
47
|
simframe baseline list recorded runs per flow, and what is committed
|
|
46
48
|
simframe hpi [device] Human Parity Index, per flow and overall
|
|
47
49
|
simframe escalations [device] why simframe handed decisions back, by reason
|
|
50
|
+
(--session=<id> narrows to one agent; the
|
|
51
|
+
ids are listed in the output. SIMFRAME_SESSION
|
|
52
|
+
names one, but only at process start — an
|
|
53
|
+
already-running MCP server cannot pick it up)
|
|
48
54
|
simframe devices list simulators
|
|
49
55
|
simframe doctor check that this machine can capture
|
|
50
56
|
(--strict, or SIMFRAME_STRICT=1, makes any
|
|
@@ -160,6 +166,25 @@ const num = (v, fallback) => (v == null ? fallback : Number(v));
|
|
|
160
166
|
* script that has to parse one command's prose and another's JSON will parse
|
|
161
167
|
* the prose wrong exactly once and then be trusted anyway.
|
|
162
168
|
*/
|
|
169
|
+
/**
|
|
170
|
+
* The machine-readable half of a failure.
|
|
171
|
+
*
|
|
172
|
+
* A refusal recognisable only by reading its prose is a refusal nobody can
|
|
173
|
+
* depend on. Our own CI asserted the stale-ref guard by matching three
|
|
174
|
+
* phrasings and went red when a fourth arrived — a *better* one, naming the
|
|
175
|
+
* label the number stood for. The sentence is for a person; these fields are
|
|
176
|
+
* the contract, and they live in one function because `find` reports its own
|
|
177
|
+
* failures and the top-level handler reports the rest.
|
|
178
|
+
*/
|
|
179
|
+
function failureJson(err) {
|
|
180
|
+
return {
|
|
181
|
+
ok: false,
|
|
182
|
+
error: err.message,
|
|
183
|
+
reason: metrics.escalationOf(err)?.reason ?? null,
|
|
184
|
+
...(err.staleRef ? { staleRef: true, staleKind: err.staleKind ?? null, staleLabel: err.staleLabel ?? null } : {}),
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
163
188
|
function emit(flags, json, lines) {
|
|
164
189
|
if (flags.json) {
|
|
165
190
|
console.log(JSON.stringify(json, null, 2));
|
|
@@ -169,11 +194,26 @@ function emit(flags, json, lines) {
|
|
|
169
194
|
if (body != null) console.log(Array.isArray(body) ? body.filter((l) => l != null).join('\n') : body);
|
|
170
195
|
}
|
|
171
196
|
|
|
172
|
-
/**
|
|
173
|
-
|
|
197
|
+
/**
|
|
198
|
+
* The end-state screen map, re-read rather than recalled, with its hint.
|
|
199
|
+
*
|
|
200
|
+
* Both halves were reported against Phase 11.5 and both were right. The map was
|
|
201
|
+
* rendered from whatever reading the flow already had, which is memory-first —
|
|
202
|
+
* so a trailing map could describe the screen as it was seconds ago, and the
|
|
203
|
+
* remedy in practice was a `ui --refresh` after nearly every call, which is a
|
|
204
|
+
* whole extra turn to save a few hundred milliseconds. Wrong way round.
|
|
205
|
+
*
|
|
206
|
+
* And the hint was only ever printed by the MCP server, so no CLI user could
|
|
207
|
+
* see it and no CLI run could test it.
|
|
208
|
+
*/
|
|
209
|
+
async function mapText(device, options, identity, { flowOk = true, escalated = false, refresh = true } = {}) {
|
|
174
210
|
try {
|
|
175
|
-
const m = await view.screenMap(device, {
|
|
176
|
-
|
|
211
|
+
const m = await view.screenMap(device, {
|
|
212
|
+
options,
|
|
213
|
+
refresh,
|
|
214
|
+
identity: refresh ? undefined : (identity?.entry ? identity : undefined),
|
|
215
|
+
});
|
|
216
|
+
return `${m.text}\n${view.hintFor(m, { flowOk, escalated })}`;
|
|
177
217
|
} catch (err) {
|
|
178
218
|
return `(could not read the screen: ${err.message})`;
|
|
179
219
|
}
|
|
@@ -229,6 +269,11 @@ async function main() {
|
|
|
229
269
|
if (flags.maxDim) options.maxDim = num(flags.maxDim);
|
|
230
270
|
if (flags.ringSize) options.ringSize = num(flags.ringSize);
|
|
231
271
|
if (flags.engine) options.engine = String(flags.engine);
|
|
272
|
+
// Per-command overrides for the two experiment knobs, so an A/B is an
|
|
273
|
+
// argument rather than a restart. `--sensor=ax-first`, `--planner=apple`.
|
|
274
|
+
if (flags.sensor) options.sensor = String(flags.sensor);
|
|
275
|
+
if (flags.planner) options.planner = String(flags.planner);
|
|
276
|
+
if (flags.supervisor) options.supervisor = String(flags.supervisor);
|
|
232
277
|
|
|
233
278
|
switch (command) {
|
|
234
279
|
case undefined:
|
|
@@ -310,6 +355,29 @@ async function main() {
|
|
|
310
355
|
return;
|
|
311
356
|
}
|
|
312
357
|
|
|
358
|
+
// Rebuild the daemon's HID session, and nothing else.
|
|
359
|
+
//
|
|
360
|
+
// The narrow remedy for the narrow fault. Restarting the daemon also cures
|
|
361
|
+
// a stale session and throws away the frame ring and every warm cache to do
|
|
362
|
+
// it, which is the difference between a fix and a power cycle.
|
|
363
|
+
case 'input': {
|
|
364
|
+
const what = positional[0];
|
|
365
|
+
if (what !== 'reset') throw new Error('usage: simframe input reset [--device <udid>]');
|
|
366
|
+
const dev = await resolveDevice(device);
|
|
367
|
+
const before = await input.sessionHealth(dev.udid);
|
|
368
|
+
const reset = await input.resetSession(dev.udid);
|
|
369
|
+
if (!reset) {
|
|
370
|
+
// Said plainly rather than as a success: there is no daemon holding a
|
|
371
|
+
// session to rebuild, so nothing was wrong and nothing was done.
|
|
372
|
+
console.log(`no simframed session to rebuild for ${dev.name} — the daemon is not running, or this device is not driven by it`);
|
|
373
|
+
process.exitCode = 1;
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
console.log(`rebuilt the HID session for ${dev.name}`
|
|
377
|
+
+ (before.stale ? `\n it was stale: ${before.reason}` : '\n it did not report stale; rebuilt anyway, as asked'));
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
|
|
313
381
|
case 'status': {
|
|
314
382
|
const udids = device
|
|
315
383
|
? [(await resolveDevice(device)).udid]
|
|
@@ -508,6 +576,7 @@ async function main() {
|
|
|
508
576
|
all: Boolean(flags.all),
|
|
509
577
|
refresh: Boolean(flags.refresh),
|
|
510
578
|
});
|
|
579
|
+
m.text = `${m.text}\n${view.hintFor(m)}`;
|
|
511
580
|
emit(
|
|
512
581
|
flags,
|
|
513
582
|
{
|
|
@@ -560,6 +629,7 @@ async function main() {
|
|
|
560
629
|
stableMs: num(flags.stableMs, 500),
|
|
561
630
|
timeoutMs: num(flags.timeoutMs, 8000),
|
|
562
631
|
continueOnError: Boolean(flags.continueOnError),
|
|
632
|
+
supervise: flags.supervise ? String(flags.supervise) : undefined,
|
|
563
633
|
options,
|
|
564
634
|
});
|
|
565
635
|
const saved = flags.save
|
|
@@ -567,7 +637,16 @@ async function main() {
|
|
|
567
637
|
: null;
|
|
568
638
|
// `--map=false` arrives as the string "false"; `--no-map` as true.
|
|
569
639
|
const wantMap = !flags.json && flags.noMap !== true && String(flags.map ?? 'true') !== 'false';
|
|
570
|
-
|
|
640
|
+
// Every local ruling, so a wrong one is correctable rather than
|
|
641
|
+
// mysterious — and the model's stated reason is shown as its claim.
|
|
642
|
+
for (const s_ of res.supervisions ?? []) {
|
|
643
|
+
console.log(`supervisor at step ${s_.index}: ${s_.decision} — ${s_.outcome}`
|
|
644
|
+
+ (s_.reason ? ` (it said: "${s_.reason}")` : ''));
|
|
645
|
+
}
|
|
646
|
+
const escalated = (res.results ?? []).some((r) => metrics.ESCALATING_VERDICTS.has(r.verification?.verdict));
|
|
647
|
+
const map = wantMap
|
|
648
|
+
? await mapText(flags.device, options, res.endScreen, { flowOk: res.ok, escalated })
|
|
649
|
+
: null;
|
|
571
650
|
emit(
|
|
572
651
|
flags,
|
|
573
652
|
{
|
|
@@ -787,7 +866,7 @@ async function main() {
|
|
|
787
866
|
],
|
|
788
867
|
);
|
|
789
868
|
} catch (err) {
|
|
790
|
-
emit(flags,
|
|
869
|
+
emit(flags, failureJson(err), err.message);
|
|
791
870
|
process.exitCode = 1;
|
|
792
871
|
}
|
|
793
872
|
return;
|
|
@@ -951,23 +1030,52 @@ async function main() {
|
|
|
951
1030
|
case 'escalations': {
|
|
952
1031
|
const dev = await resolveDevice(flags.device);
|
|
953
1032
|
const records = metrics.readEscalations(dev.udid, { limit: flags.last ? num(flags.last) : undefined });
|
|
954
|
-
const b = metrics.breakdown(records
|
|
1033
|
+
const b = metrics.breakdown(records, {
|
|
1034
|
+
session: flags.session === true ? metrics.sessionId() : (flags.session ? String(flags.session) : null),
|
|
1035
|
+
flow: flags.flow ? String(flags.flow) : null,
|
|
1036
|
+
});
|
|
955
1037
|
if (flags.out) store.writeAtomic(String(flags.out), `${JSON.stringify(b, null, 2)}\n`);
|
|
956
1038
|
emit(flags, b, [
|
|
957
1039
|
`${b.total} escalation${b.total === 1 ? '' : 's'} on ${dev.name}`,
|
|
958
1040
|
...metrics.REASONS
|
|
959
1041
|
.filter((r) => b.by_reason[r])
|
|
960
1042
|
.sort((a, c) => b.by_reason[c] - b.by_reason[a])
|
|
961
|
-
.map((r) => ` ${r.padEnd(20)} ${String(b.by_reason[r]).padStart(4)}
|
|
1043
|
+
.map((r) => ` ${r.padEnd(20)} ${String(b.by_reason[r]).padStart(4)} `
|
|
1044
|
+
+ (metrics.BUILT_FACULTIES.has(metrics.FACULTY[r])
|
|
1045
|
+
? `not removed by: ${metrics.FACULTY[r]} [built]`
|
|
1046
|
+
: `would be removed by: ${metrics.FACULTY[r]}`)),
|
|
962
1047
|
b.total ? '' : null,
|
|
963
1048
|
b.total ? `avoidable ${b.avoidable}/${b.total} (${b.avoidable_escalation_rate})` : null,
|
|
964
1049
|
// Said out loud rather than left for someone to discover: the rate is
|
|
965
1050
|
// 1.0 while no faculty exists, so the breakdown above is the number
|
|
966
1051
|
// that decides the next phase.
|
|
967
1052
|
b.total && b.avoidable_escalation_rate === 1
|
|
968
|
-
?
|
|
1053
|
+
? (metrics.REASONS.some((r) => b.by_reason[r] && metrics.BUILT_FACULTIES.has(metrics.FACULTY[r]))
|
|
1054
|
+
? ' the rate is 1.0 because nothing resolves locally yet. A reason marked [built] is not a queue waiting on a phase — it is evidence the phase that shipped is not sufficient.'
|
|
1055
|
+
: ' every reason maps to a faculty that is not built yet, so this rate is 1.0 by construction. The per-reason counts are the steering wheel.')
|
|
969
1056
|
: null,
|
|
970
1057
|
b.total ? `model turns spent on escalations: ${b.model_turns_spent}` : null,
|
|
1058
|
+
// The log is per-device and shared. Said before the counts are used,
|
|
1059
|
+
// not after: two agents on one booted simulator write one interleaved
|
|
1060
|
+
// file, and CLAUDE.md makes these counts the thing that picks the next
|
|
1061
|
+
// faculty. A pooled breakdown errs toward whichever session made more
|
|
1062
|
+
// mistakes, which is a different question.
|
|
1063
|
+
b.pooled
|
|
1064
|
+
? 'WARNING these counts may pool more than one agent\'s work: '
|
|
1065
|
+
+ [
|
|
1066
|
+
b.session_count > 1 ? `${b.session_count} sessions` : null,
|
|
1067
|
+
b.unattributed ? `${b.unattributed} record(s) written before sessions were logged` : null,
|
|
1068
|
+
].filter(Boolean).join(', ')
|
|
1069
|
+
+ '. Narrow with --session (this process), --session=<id>, or --flow=<name>.'
|
|
1070
|
+
: null,
|
|
1071
|
+
b.session_count > 1 ? 'sessions:' : null,
|
|
1072
|
+
...(b.session_count > 1
|
|
1073
|
+
? b.sessions.map((x) => ` ${x.session_id.padEnd(22)} ${String(x.count).padStart(4)} ${x.client}`)
|
|
1074
|
+
: []),
|
|
1075
|
+
Object.keys(b.by_flow).length > 1 ? 'flows:' : null,
|
|
1076
|
+
...(Object.keys(b.by_flow).length > 1
|
|
1077
|
+
? Object.entries(b.by_flow).slice(0, 10).map(([n, c]) => ` ${n.padEnd(28)} ${String(c).padStart(4)}`)
|
|
1078
|
+
: []),
|
|
971
1079
|
b.top_screens.length ? 'top screens:' : null,
|
|
972
1080
|
...b.top_screens.map((s) => ` ${s.fingerprint.slice(0, 16).padEnd(18)} ${s.count}`),
|
|
973
1081
|
metrics.writeError() ? `WARNING a log write failed: ${metrics.writeError()}` : null,
|
|
@@ -993,6 +1101,10 @@ async function main() {
|
|
|
993
1101
|
json: Boolean(flags.json),
|
|
994
1102
|
strict: Boolean(flags.strict) || process.env.SIMFRAME_STRICT === '1',
|
|
995
1103
|
device: flags.device,
|
|
1104
|
+
// So `doctor --sensor=ax-first --planner=apple` reports the mode the
|
|
1105
|
+
// caller is about to use, not the one the environment happens to hold.
|
|
1106
|
+
// Confirming the mode before a run is the whole reason to read this.
|
|
1107
|
+
options,
|
|
996
1108
|
});
|
|
997
1109
|
return;
|
|
998
1110
|
}
|
|
@@ -1069,7 +1181,7 @@ async function blackScreenProbe(udid) {
|
|
|
1069
1181
|
}
|
|
1070
1182
|
}
|
|
1071
1183
|
|
|
1072
|
-
async function doctor({ json = false, strict = false, device } = {}) {
|
|
1184
|
+
async function doctor({ json = false, strict = false, device, options = {} } = {}) {
|
|
1073
1185
|
const checks = [];
|
|
1074
1186
|
// `level` is 'ok' | 'warn' | 'fail'. A warn means it works but not the way it
|
|
1075
1187
|
// should — the exact state that used to be invisible.
|
|
@@ -1118,6 +1230,55 @@ async function doctor({ json = false, strict = false, device } = {}) {
|
|
|
1118
1230
|
add('on-device OCR', 'warn', err.message, { key: 'ocr.available', value: false });
|
|
1119
1231
|
}
|
|
1120
1232
|
|
|
1233
|
+
// Which sensors a read asks for, and what the recognition-level flag can and
|
|
1234
|
+
// cannot reach. Stated because `SIMFRAME_OCR` looks like it configures the
|
|
1235
|
+
// OCR everyone uses and does not: the daemon reads text in-process off the
|
|
1236
|
+
// framebuffer and owns its own recognition level, so the flag only reaches
|
|
1237
|
+
// the no-daemon fallback in `native/ocr.swift`.
|
|
1238
|
+
try {
|
|
1239
|
+
const api = await import('./index.js');
|
|
1240
|
+
const ocrMod = await import('./ocr.js');
|
|
1241
|
+
const mode = api.sensorMode(options);
|
|
1242
|
+
add('sensor mode', 'ok',
|
|
1243
|
+
mode === 'ax-first'
|
|
1244
|
+
? 'ax-first — the tree alone (~50ms), paying for OCR only when a resolve fails'
|
|
1245
|
+
: 'full — accessibility and OCR fused on every read (~164ms)',
|
|
1246
|
+
{ key: 'sensor.mode', value: mode });
|
|
1247
|
+
add('OCR level', 'ok', `${ocrMod.level()} (fallback helper only; the daemon owns its own)`, {
|
|
1248
|
+
key: 'ocr.level',
|
|
1249
|
+
value: ocrMod.level(),
|
|
1250
|
+
});
|
|
1251
|
+
} catch { /* reported by the layers above */ }
|
|
1252
|
+
|
|
1253
|
+
// The local supervisor. Behind the hands and in front of the reasoner, and
|
|
1254
|
+
// able to say only wait/retry/stop.
|
|
1255
|
+
try {
|
|
1256
|
+
const supervisor = await import('./supervisor.js');
|
|
1257
|
+
const st = await supervisor.status(options);
|
|
1258
|
+
add('local supervisor', 'ok', `${st.supervisor} — ${st.detail}`, {
|
|
1259
|
+
key: 'supervisor.backend',
|
|
1260
|
+
value: st.supervisor,
|
|
1261
|
+
});
|
|
1262
|
+
supervisor.close();
|
|
1263
|
+
} catch (err) {
|
|
1264
|
+
add('local supervisor', 'ok', `none — ${err.message}`, { key: 'supervisor.backend', value: 'none' });
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
// The local planner tier. `none` is the normal answer and not a fault: it is
|
|
1268
|
+
// off unless SIMFRAME_PLANNER asks for it, and it only ever reorders
|
|
1269
|
+
// candidates that exploration was going to try anyway.
|
|
1270
|
+
try {
|
|
1271
|
+
const planner = await import('./planner.js');
|
|
1272
|
+
const st = await planner.status(options);
|
|
1273
|
+
add('local planner', 'ok', `${st.planner} — ${st.detail}`, {
|
|
1274
|
+
key: 'planner.backend',
|
|
1275
|
+
value: st.planner,
|
|
1276
|
+
});
|
|
1277
|
+
planner.close();
|
|
1278
|
+
} catch (err) {
|
|
1279
|
+
add('local planner', 'ok', `none — ${err.message}`, { key: 'planner.backend', value: 'none' });
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1121
1282
|
try {
|
|
1122
1283
|
let booted = await bootedDevices();
|
|
1123
1284
|
// Respect --device. Without this, doctor reports on every booted simulator,
|
|
@@ -1228,7 +1389,18 @@ async function doctor({ json = false, strict = false, device } = {}) {
|
|
|
1228
1389
|
// dispatched successfully and moved nothing, five runs in a row.
|
|
1229
1390
|
const session = await input.sessionHealth(d.udid);
|
|
1230
1391
|
if (session.stale) {
|
|
1231
|
-
|
|
1392
|
+
// The remedy used to read "the next action rebuilds it automatically;
|
|
1393
|
+
// simframe stop && simframe start does it now", and both halves were
|
|
1394
|
+
// wrong. The first was false wherever it mattered, because the rebuild
|
|
1395
|
+
// check was gated once per process and the MCP server is one process
|
|
1396
|
+
// for a whole session. The second names a command that fails twice:
|
|
1397
|
+
// `stop` needs `--device` when two simulators are booted, and then
|
|
1398
|
+
// refuses because a client holds the daemon, so the sequence that
|
|
1399
|
+
// actually works is `stop --device <udid> --force && start --device
|
|
1400
|
+
// <udid>` — a daemon restart, to fix a session, when rebuilding the
|
|
1401
|
+
// session is a thing the daemon can already do on request. It just had
|
|
1402
|
+
// no way in from outside. It does now.
|
|
1403
|
+
add(`input session (${d.name})`, 'warn', `stale — ${session.reason}. The next action rebuilds it; simframe input reset --device ${d.udid} does it now`,
|
|
1232
1404
|
{ key: 'input.session', value: 'stale' });
|
|
1233
1405
|
} else if (caps.input.supported) {
|
|
1234
1406
|
add(`input session (${d.name})`, 'ok', session.reason ?? 'current with this device session',
|
|
@@ -1252,8 +1424,20 @@ async function doctor({ json = false, strict = false, device } = {}) {
|
|
|
1252
1424
|
if (probed.length) {
|
|
1253
1425
|
const t0 = Date.now();
|
|
1254
1426
|
const res = await api.getFrame(probed[0].udid);
|
|
1255
|
-
|
|
1256
|
-
|
|
1427
|
+
// The wedge's whole signature is that everything here reports success.
|
|
1428
|
+
// A black frame is 32 integer comparisons on a signature already
|
|
1429
|
+
// computed, and it is what separates "captured a frame" from "captured
|
|
1430
|
+
// a frame of a display that has stopped rendering".
|
|
1431
|
+
const dark = analyze.isBlackFrame(
|
|
1432
|
+
(res.state.history ?? []).find((h) => h.seq === res.state.seq)?.sig ?? null,
|
|
1433
|
+
);
|
|
1434
|
+
add('capture', dark ? 'warn' : 'ok',
|
|
1435
|
+
`frame #${res.state.seq} ${res.width}x${res.height} in ${Date.now() - t0}ms (age ${res.ageMs}ms)`
|
|
1436
|
+
+ (dark
|
|
1437
|
+
? ' — and every pixel of it is black. If the device is not showing a black screen on purpose,'
|
|
1438
|
+
+ ' this is the display pipeline having stopped rendering; it usually recovers on its own,'
|
|
1439
|
+
+ ` and ${'xcrun simctl shutdown'} / boot is the cure that always works.`
|
|
1440
|
+
: ''),
|
|
1257
1441
|
{ key: 'capture.frames', value: res.state.seq });
|
|
1258
1442
|
// A wedged device produces the same nothing as a quiet one, so doctor has
|
|
1259
1443
|
// to ask the capture loop rather than look at the frames. `fail`, not
|
|
@@ -1321,13 +1505,28 @@ async function doctor({ json = false, strict = false, device } = {}) {
|
|
|
1321
1505
|
process.exitCode = failed.length || (strict && warned.length) ? 1 : 0;
|
|
1322
1506
|
}
|
|
1323
1507
|
|
|
1324
|
-
|
|
1508
|
+
// A long-lived local helper must not decide when the CLI exits. It is closed
|
|
1509
|
+
// after every command, whether or not one was ever started — `close()` on an
|
|
1510
|
+
// unopened planner is a no-op, and leaving it open made a finished flow hang.
|
|
1511
|
+
const closeHelpers = async () => {
|
|
1512
|
+
try {
|
|
1513
|
+
const planner = await import('./planner.js');
|
|
1514
|
+
planner.close();
|
|
1515
|
+
const supervisor = await import('./supervisor.js');
|
|
1516
|
+
supervisor.close();
|
|
1517
|
+
} catch { /* nothing to close */ }
|
|
1518
|
+
};
|
|
1519
|
+
|
|
1520
|
+
main().then(closeHelpers, async (err) => {
|
|
1521
|
+
await closeHelpers();
|
|
1522
|
+
throw err;
|
|
1523
|
+
}).catch((err) => {
|
|
1325
1524
|
// A caller that asked for JSON gets JSON, failures included. Printing prose
|
|
1326
1525
|
// here handed `JSON.parse` a SyntaxError instead of a reason, so a script
|
|
1327
1526
|
// could not tell "the daemon lost the display" from "simframe is broken" —
|
|
1328
1527
|
// which is the whole point of a machine-readable interface.
|
|
1329
1528
|
if (process.argv.includes('--json')) {
|
|
1330
|
-
process.stdout.write(`${JSON.stringify(
|
|
1529
|
+
process.stdout.write(`${JSON.stringify(failureJson(err), null, 2)}\n`);
|
|
1331
1530
|
} else {
|
|
1332
1531
|
process.stderr.write(`simframe: ${err.message}\n`);
|
|
1333
1532
|
}
|
package/src/control.js
CHANGED
|
@@ -65,6 +65,7 @@ export const swipe = (udid, from, to, opts = {}) =>
|
|
|
65
65
|
export const type = (udid, text) => request(udid, { action: 'type', text });
|
|
66
66
|
export const paste = (udid, text) => request(udid, { action: 'paste', text });
|
|
67
67
|
export const press = (udid, button) => request(udid, { action: 'press', button });
|
|
68
|
+
export const key = (udid, usage, modifiers = []) => request(udid, { action: 'key', usage, modifiers });
|
|
68
69
|
export const status = (udid) => request(udid, { action: 'status' });
|
|
69
70
|
export const resetInput = (udid) => request(udid, { action: 'resetInput' });
|
|
70
71
|
export const longPress = (udid, x, y, opts = {}) => request(udid, { action: 'longPress', x, y, ...opts });
|
package/src/fingerprint.js
CHANGED
|
@@ -30,8 +30,26 @@ import * as regions from './regions.js';
|
|
|
30
30
|
* list screen. Same rules, different input, therefore different hashes —
|
|
31
31
|
* and a stored hash that can never match again is the quietest kind of
|
|
32
32
|
* wrong, which is what this counter exists to prevent.
|
|
33
|
+
* 5 — a phantom keyboard was deleting screens' content from their identity. A
|
|
34
|
+
* dozen short text rows of uniform height stacked low on a read-only
|
|
35
|
+
* summary satisfied every size-and-uniformity test for a keyboard, and
|
|
36
|
+
* `tokens` discards everything below `keyboardTop` — so two screens of one
|
|
37
|
+
* wizard, sharing a nav title and a step indicator, collapsed onto a single
|
|
38
|
+
* hash. `detectKeyboardTop` now requires the small uniform boxes to be
|
|
39
|
+
* key-shaped. Every screen with content in its lower half hashes
|
|
40
|
+
* differently, so the stored graph and maps must go.
|
|
41
|
+
* 6 — the opposite half of the same bug, and it took a recorded screen to see.
|
|
42
|
+
* `KEYBOARD_MIN_FRACTION` is a *detection window*, not a keyboard's height,
|
|
43
|
+
* and its edge was being used as the boundary — so on an iPhone 17 Pro with
|
|
44
|
+
* the software keyboard up, the window starts at y=629 while the `q`–`p`
|
|
45
|
+
* row's frame top is **590**, and that whole row fell outside it. Ten
|
|
46
|
+
* keyboard keys were reported as page content and counted into the screen's
|
|
47
|
+
* identity, in the same map that said `keyboard up`. The boundary now
|
|
48
|
+
* extends upward while the rows above keep being key-shaped, which page
|
|
49
|
+
* content is not. Any screen fingerprinted with a keyboard up hashes
|
|
50
|
+
* differently, so the stored graph and maps must go again.
|
|
33
51
|
*/
|
|
34
|
-
export const TOKEN_RULES_VERSION =
|
|
52
|
+
export const TOKEN_RULES_VERSION = 6;
|
|
35
53
|
|
|
36
54
|
/** Frames are quantised to this, so sub-pixel drift and a nudged row do not matter. */
|
|
37
55
|
export const GRID = 24;
|