simframe 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -3
- package/flows/hpi-suite.json +68 -0
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +49 -1
- package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +7 -0
- package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +11 -0
- package/native/simframed/Sources/SimframeCore/CaptureRecovery.swift +48 -0
- package/native/simframed/Sources/simframed/main.swift +128 -73
- package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +40 -0
- package/package.json +2 -1
- package/scripts/bench-hpi.mjs +254 -0
- package/scripts/check-package.mjs +7 -0
- package/scripts/check-private.mjs +143 -0
- package/scripts/eval-perception.mjs +248 -0
- package/src/actions.js +333 -14
- package/src/analyze.js +70 -0
- package/src/baseline.js +333 -0
- package/src/cli.js +410 -4
- package/src/daemon.js +9 -0
- package/src/fingerprint.js +7 -1
- package/src/graph.js +262 -1
- package/src/index.js +335 -22
- package/src/input.js +155 -1
- package/src/intent.js +11 -2
- package/src/matching.js +136 -4
- package/src/mcp.js +14 -1
- package/src/metrics.js +596 -0
- package/src/navigate.js +47 -7
- package/src/platform/android.js +16 -1
- package/src/platform/index.js +3 -0
- package/src/platform/ios.js +40 -0
- package/src/screenmap.js +55 -14
- package/src/view.js +65 -3
package/src/cli.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
3
4
|
import path from 'node:path';
|
|
4
5
|
import { runDaemon, DEFAULTS } from './daemon.js';
|
|
5
|
-
import { bootedDevices, capabilitiesFor, listDevices, PLATFORMS, resolveDevice, toolchainChecks } from './platform/index.js';
|
|
6
|
+
import { bootedDevices, capabilitiesFor, listDevices, PLATFORMS, resolveDevice, screenshot, toolchainChecks } from './platform/index.js';
|
|
6
7
|
import * as actions from './actions.js';
|
|
8
|
+
import * as analyze from './analyze.js';
|
|
7
9
|
import * as api from './index.js';
|
|
8
10
|
import * as input from './input.js';
|
|
11
|
+
import * as baseline from './baseline.js';
|
|
12
|
+
import * as metrics from './metrics.js';
|
|
9
13
|
import * as navigate from './navigate.js';
|
|
14
|
+
import { decodePng } from './png.js';
|
|
10
15
|
import * as store from './store.js';
|
|
11
16
|
import * as view from './view.js';
|
|
12
17
|
|
|
@@ -16,6 +21,7 @@ const USAGE = `simframe — always-warm iOS Simulator frames
|
|
|
16
21
|
simframe start [device] start the capture loop in the background
|
|
17
22
|
simframe stop [device|--all] stop the capture loop
|
|
18
23
|
simframe status [device] show daemon and newest-frame status
|
|
24
|
+
simframe input reset rebuild the HID session (see doctor)
|
|
19
25
|
simframe frame [device] write the newest frame to a file
|
|
20
26
|
simframe state [device] print frame metadata and the change map
|
|
21
27
|
simframe mark [device] print the current frame hash, to use as --since
|
|
@@ -36,6 +42,11 @@ const USAGE = `simframe — always-warm iOS Simulator frames
|
|
|
36
42
|
simframe type <text> enter text (exact; uses the pasteboard)
|
|
37
43
|
simframe keys <text> send key events instead (layout-dependent)
|
|
38
44
|
simframe press <button> a hardware button, e.g. home
|
|
45
|
+
simframe baseline record <flow> record a human performing a flow (see below)
|
|
46
|
+
simframe baseline summarize <flow> write the median/IQR baseline for it
|
|
47
|
+
simframe baseline list recorded runs per flow, and what is committed
|
|
48
|
+
simframe hpi [device] Human Parity Index, per flow and overall
|
|
49
|
+
simframe escalations [device] why simframe handed decisions back, by reason
|
|
39
50
|
simframe devices list simulators
|
|
40
51
|
simframe doctor check that this machine can capture
|
|
41
52
|
(--strict, or SIMFRAME_STRICT=1, makes any
|
|
@@ -46,6 +57,21 @@ Selectors — anywhere a control is named
|
|
|
46
57
|
"Save" a label or a phrase, resolved by intent (verbs, typos, synonyms)
|
|
47
58
|
@120,400 raw point coordinates
|
|
48
59
|
|
|
60
|
+
Measuring against a human — the Human Parity Index
|
|
61
|
+
|
|
62
|
+
A flow's agent time is measured every time it runs; the human half has to be
|
|
63
|
+
recorded once, by a person, on the same simulator:
|
|
64
|
+
|
|
65
|
+
simframe baseline record settings-larger-text --device=<udid> --runs=5
|
|
66
|
+
simframe baseline summarize settings-larger-text
|
|
67
|
+
simframe hpi --device=<udid>
|
|
68
|
+
|
|
69
|
+
\`record\` puts the device on the home screen, waits for you to start, and
|
|
70
|
+
waits again for you to stop. Wall time is measured between those two; the
|
|
71
|
+
step count is derived from screen transitions, because a human tapping the
|
|
72
|
+
Simulator window leaves no HID log to read. Five runs is the recommendation
|
|
73
|
+
and three is the floor. The flows live in flows/hpi-suite.json.
|
|
74
|
+
|
|
49
75
|
Options
|
|
50
76
|
--device=<udid|name> simulator to target (default: the booted one)
|
|
51
77
|
--json machine-readable output — on every command
|
|
@@ -96,6 +122,7 @@ The reliable pattern around an action is:
|
|
|
96
122
|
const VALUE_FLAGS = new Set([
|
|
97
123
|
'ago', 'count', 'detail', 'device', 'durationMs', 'engine', 'filter', 'fps', 'index', 'maxDim',
|
|
98
124
|
'mode', 'out', 'ringSize', 'since', 'spanMs', 'stableMs', 'timeoutMs',
|
|
125
|
+
'last', 'runs', 'suite', 'keepLast', 'reason',
|
|
99
126
|
]);
|
|
100
127
|
|
|
101
128
|
function parseArgs(argv) {
|
|
@@ -154,6 +181,40 @@ async function mapText(device, options, identity) {
|
|
|
154
181
|
}
|
|
155
182
|
}
|
|
156
183
|
|
|
184
|
+
/**
|
|
185
|
+
* Wait for Enter, on a terminal or on a pipe.
|
|
186
|
+
*
|
|
187
|
+
* `readline/promises` looked like the obvious choice and threw "readline was
|
|
188
|
+
* closed" the first time it was asked a question after piped input ran out —
|
|
189
|
+
* which is how a smoke test of `baseline record` would have handed a stack
|
|
190
|
+
* trace to the person recording. Buffered lines are queued, and a closed
|
|
191
|
+
* stream is reported as what it is rather than thrown from inside a library.
|
|
192
|
+
*/
|
|
193
|
+
async function lineReader() {
|
|
194
|
+
const readline = await import('node:readline');
|
|
195
|
+
const rl = readline.createInterface({ input: process.stdin, terminal: Boolean(process.stdin.isTTY) });
|
|
196
|
+
const queue = [];
|
|
197
|
+
const waiters = [];
|
|
198
|
+
let closed = false;
|
|
199
|
+
rl.on('line', (line) => (waiters.length ? waiters.shift()({ line }) : queue.push(line)));
|
|
200
|
+
rl.on('close', () => {
|
|
201
|
+
closed = true;
|
|
202
|
+
while (waiters.length) waiters.shift()({ closed: true });
|
|
203
|
+
});
|
|
204
|
+
return {
|
|
205
|
+
async enter(prompt) {
|
|
206
|
+
process.stdout.write(prompt);
|
|
207
|
+
const got = queue.length ? { line: queue.shift() } : closed ? { closed: true } : await new Promise((r) => waiters.push(r));
|
|
208
|
+
if (got.closed) {
|
|
209
|
+
throw new Error('stdin closed before the run ended — baseline record needs an interactive terminal');
|
|
210
|
+
}
|
|
211
|
+
process.stdout.write('\n');
|
|
212
|
+
return got.line;
|
|
213
|
+
},
|
|
214
|
+
close: () => rl.close(),
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
|
|
157
218
|
/** A step result, the same shape in every command that runs steps. */
|
|
158
219
|
const stepLine = (r) => {
|
|
159
220
|
const settle = r.settled ? (r.settled.ok ? ` (settled ${r.settled.waitedMs}ms)` : ' (never settled)') : '';
|
|
@@ -251,6 +312,29 @@ async function main() {
|
|
|
251
312
|
return;
|
|
252
313
|
}
|
|
253
314
|
|
|
315
|
+
// Rebuild the daemon's HID session, and nothing else.
|
|
316
|
+
//
|
|
317
|
+
// The narrow remedy for the narrow fault. Restarting the daemon also cures
|
|
318
|
+
// a stale session and throws away the frame ring and every warm cache to do
|
|
319
|
+
// it, which is the difference between a fix and a power cycle.
|
|
320
|
+
case 'input': {
|
|
321
|
+
const what = positional[0];
|
|
322
|
+
if (what !== 'reset') throw new Error('usage: simframe input reset [--device <udid>]');
|
|
323
|
+
const dev = await resolveDevice(device);
|
|
324
|
+
const before = await input.sessionHealth(dev.udid);
|
|
325
|
+
const reset = await input.resetSession(dev.udid);
|
|
326
|
+
if (!reset) {
|
|
327
|
+
// Said plainly rather than as a success: there is no daemon holding a
|
|
328
|
+
// session to rebuild, so nothing was wrong and nothing was done.
|
|
329
|
+
console.log(`no simframed session to rebuild for ${dev.name} — the daemon is not running, or this device is not driven by it`);
|
|
330
|
+
process.exitCode = 1;
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
console.log(`rebuilt the HID session for ${dev.name}`
|
|
334
|
+
+ (before.stale ? `\n it was stale: ${before.reason}` : '\n it did not report stale; rebuilt anyway, as asked'));
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
|
|
254
338
|
case 'status': {
|
|
255
339
|
const udids = device
|
|
256
340
|
? [(await resolveDevice(device)).udid]
|
|
@@ -305,13 +389,23 @@ async function main() {
|
|
|
305
389
|
}
|
|
306
390
|
|
|
307
391
|
case 'state': {
|
|
308
|
-
const res = await api.getState(device, { since: flags.since, options });
|
|
392
|
+
const res = await api.getState(device, { since: flags.since, options, inputHealth: true });
|
|
309
393
|
if (flags.json) {
|
|
310
394
|
console.log(JSON.stringify({ ...res.state, history: undefined, ageMs: res.ageMs, since: res.since, live: res.live }, null, 2));
|
|
311
395
|
} else {
|
|
312
396
|
const s = res.state;
|
|
313
397
|
const out = [];
|
|
314
398
|
if (!res.live.ok) out.push(`WARNING: ${res.live.note}`);
|
|
399
|
+
// A cause, rather than five silent no-ops. Every tap on a stale
|
|
400
|
+
// session is dispatched successfully and moves nothing.
|
|
401
|
+
if (res.input?.stale) out.push(`input: stale — ${res.input.reason}`);
|
|
402
|
+
if (res.timing?.samples) {
|
|
403
|
+
out.push(
|
|
404
|
+
`timing: this screen usually arrives in ${res.timing.edge_p50}ms (p95 ${res.timing.edge_p95}ms, `
|
|
405
|
+
+ `${res.timing.samples} samples); ${res.timing.elapsed_ms}ms since the last change`
|
|
406
|
+
+ (res.timing.slower_than_usual ? ` — ${res.timing.note}` : ''),
|
|
407
|
+
);
|
|
408
|
+
}
|
|
315
409
|
out.push(`${res.device.name} frame #${s.seq} age ${res.ageMs}ms ${s.width}x${s.height}`);
|
|
316
410
|
out.push(`hash ${s.hash} stable ${s.stableForMs}ms`);
|
|
317
411
|
if (res.since?.kind === 'history') {
|
|
@@ -724,6 +818,217 @@ async function main() {
|
|
|
724
818
|
return;
|
|
725
819
|
}
|
|
726
820
|
|
|
821
|
+
case 'baseline': {
|
|
822
|
+
const [sub, name] = positional;
|
|
823
|
+
const suite = baseline.loadSuite(flags.suite ?? baseline.SUITE_FILE);
|
|
824
|
+
|
|
825
|
+
if (sub === 'list' || sub == null) {
|
|
826
|
+
const dev = await resolveDevice(flags.device);
|
|
827
|
+
const committed = baseline.readBaselines();
|
|
828
|
+
const rows = suite.map((f) => {
|
|
829
|
+
const runs = baseline.readRuns(dev.udid, f.name);
|
|
830
|
+
return {
|
|
831
|
+
flow: f.name,
|
|
832
|
+
recorded_runs: runs.length,
|
|
833
|
+
committed: Boolean(committed[f.name]),
|
|
834
|
+
human_median_ms: committed[f.name]?.wall_time_ms?.p50 ?? null,
|
|
835
|
+
min_steps: f.minSteps ?? null,
|
|
836
|
+
};
|
|
837
|
+
});
|
|
838
|
+
emit(flags, rows, rows.map((r) =>
|
|
839
|
+
`${r.flow.padEnd(24)} ${String(r.recorded_runs).padStart(2)} run${r.recorded_runs === 1 ? ' ' : 's'}` +
|
|
840
|
+
` ${r.committed ? `committed, human p50 ${r.human_median_ms}ms` : 'not committed'}`));
|
|
841
|
+
return;
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
if (sub === 'exclude') {
|
|
845
|
+
if (!name) throw new Error('usage: simframe baseline exclude <flow> --keep-last=<n> [--reason="..."]');
|
|
846
|
+
const dev = await resolveDevice(flags.device);
|
|
847
|
+
baseline.flowFrom(suite, name);
|
|
848
|
+
const keepLast = num(flags.keepLast, NaN);
|
|
849
|
+
if (!Number.isFinite(keepLast)) throw new Error('pass --keep-last=<n>: how many of the most recent runs to keep');
|
|
850
|
+
const res = baseline.excludeRuns(dev.udid, name, { keepLast, reason: flags.reason ? String(flags.reason) : undefined });
|
|
851
|
+
emit(flags, res, `${name}: ${res.excluded} of ${res.total} run(s) marked excluded — the runs stay in the log, carrying why`);
|
|
852
|
+
return;
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
if (sub === 'summarize') {
|
|
856
|
+
if (!name) throw new Error('usage: simframe baseline summarize <flow>');
|
|
857
|
+
const dev = await resolveDevice(flags.device);
|
|
858
|
+
const flow = baseline.flowFrom(suite, name);
|
|
859
|
+
const runs = baseline.readRuns(dev.udid, name);
|
|
860
|
+
const res = baseline.summarizeRuns(name, runs, { minSteps: flow.minSteps ?? null, device: dev.udid });
|
|
861
|
+
if (!res.ok) {
|
|
862
|
+
emit(flags, res, `${runs.length} recorded run${runs.length === 1 ? '' : 's'} for "${name}" — ` +
|
|
863
|
+
`${baseline.MIN_RUNS} is the floor and ${baseline.WANT_RUNS} is the recommendation. ` +
|
|
864
|
+
`Record more: simframe baseline record ${name} --device=${dev.udid}`);
|
|
865
|
+
process.exitCode = 1;
|
|
866
|
+
return;
|
|
867
|
+
}
|
|
868
|
+
const file = flags.out ? baseline.writeSummary(res.summary, { dir: path.dirname(flags.out) }) : baseline.writeSummary(res.summary);
|
|
869
|
+
const w = res.summary.wall_time_ms;
|
|
870
|
+
emit(flags, { ...res.summary, file }, [
|
|
871
|
+
`${name}: ${res.summary.runs} runs`,
|
|
872
|
+
` wall time p50 ${w.p50}ms IQR ${w.p25}–${w.p75}ms range ${w.min}–${w.max}ms`,
|
|
873
|
+
` steps p50 ${res.summary.steps_observed.p50} (from screen transitions; min_steps ${res.summary.min_steps ?? '?'} from the flow)`,
|
|
874
|
+
res.summary.runs_with_incomplete_history
|
|
875
|
+
? ` WARNING ${res.summary.runs_with_incomplete_history} run(s) outran the 90s frame history; their step counts are undercounts`
|
|
876
|
+
: null,
|
|
877
|
+
` wrote ${file}`,
|
|
878
|
+
]);
|
|
879
|
+
return;
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
if (sub === 'record') {
|
|
883
|
+
if (!name) throw new Error('usage: simframe baseline record <flow>');
|
|
884
|
+
const flow = baseline.flowFrom(suite, name);
|
|
885
|
+
const dev = await resolveDevice(flags.device);
|
|
886
|
+
const wanted = Math.max(1, num(flags.runs, 1));
|
|
887
|
+
const rl = await lineReader();
|
|
888
|
+
const done = [];
|
|
889
|
+
try {
|
|
890
|
+
console.log(`${name} on ${dev.name} (${dev.udid})`);
|
|
891
|
+
console.log(`${flow.note ?? ''}\n`);
|
|
892
|
+
console.log('Do this, at your natural pace:');
|
|
893
|
+
for (const [i, line] of (flow.human ?? []).entries()) console.log(` ${i + 1}. ${line}`);
|
|
894
|
+
console.log(`\nThe shortest route is ${flow.minSteps} steps. Practise once or twice first —`);
|
|
895
|
+
console.log('a baseline should measure a tester who knows the flow, not one discovering it.\n');
|
|
896
|
+
for (let run = 1; run <= wanted; run += 1) {
|
|
897
|
+
const reset = await baseline.resetFor(dev.udid, flow);
|
|
898
|
+
if (reset.failures.length) console.log(` (reset: ${reset.failures.join('; ')})`);
|
|
899
|
+
await rl.enter(`run ${run}/${wanted} — device is on the home screen. Press Enter, then do the flow: `);
|
|
900
|
+
const res = await baseline.recordHumanRun(dev.udid, name, {
|
|
901
|
+
suite,
|
|
902
|
+
options,
|
|
903
|
+
waitForStop: () => rl.enter(` timing... press Enter the moment you are on "${flow.endsOn ?? 'the last screen'}": `),
|
|
904
|
+
});
|
|
905
|
+
done.push(res.run);
|
|
906
|
+
const r = res.run;
|
|
907
|
+
console.log(` ${r.wall_time_ms}ms, ${r.steps_observed} screen transitions` +
|
|
908
|
+
(r.history_complete === false ? ' — WARNING: longer than the frame history, steps undercounted' : ''));
|
|
909
|
+
}
|
|
910
|
+
} finally {
|
|
911
|
+
rl.close();
|
|
912
|
+
}
|
|
913
|
+
const total = baseline.readRuns(dev.udid, name).length;
|
|
914
|
+
emit(flags, { flow: name, recorded: done, runs_on_file: total }, [
|
|
915
|
+
'',
|
|
916
|
+
`${done.length} run${done.length === 1 ? '' : 's'} recorded — ${total} on file for "${name}"`,
|
|
917
|
+
total < baseline.WANT_RUNS
|
|
918
|
+
? `${baseline.WANT_RUNS - total} more would meet the recommendation; ${Math.max(0, baseline.MIN_RUNS - total)} more is the floor`
|
|
919
|
+
: `enough to summarize: simframe baseline summarize ${name} --device=${dev.udid}`,
|
|
920
|
+
]);
|
|
921
|
+
return;
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
throw new Error('usage: simframe baseline <record|summarize|list> [flow]');
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
case 'hpi': {
|
|
928
|
+
const dev = await resolveDevice(flags.device);
|
|
929
|
+
const suite = baseline.loadSuite(flags.suite ?? baseline.SUITE_FILE);
|
|
930
|
+
const names = new Set(suite.map((f) => f.name));
|
|
931
|
+
const all = metrics.readFlows(dev.udid);
|
|
932
|
+
// Named runs only, and only flows this suite defines. An ad-hoc `sim_do`
|
|
933
|
+
// is timed and logged, but it has no human counterpart and averaging it
|
|
934
|
+
// into a parity index would be inventing a comparison.
|
|
935
|
+
let runs = all.filter((f) => f.flow_name && names.has(f.flow_name) && (!flags.flow || f.flow_name === flags.flow));
|
|
936
|
+
// `--last=n` keeps the n most recent runs of each flow. The log is
|
|
937
|
+
// append-only on purpose — it is the trend — but a local log carries
|
|
938
|
+
// runs from a wedged device and from a bug since fixed, and averaging
|
|
939
|
+
// those into today's number describes neither day. CI computes its
|
|
940
|
+
// number from one process's own runs and never needs this.
|
|
941
|
+
if (flags.last) {
|
|
942
|
+
const keep = num(flags.last);
|
|
943
|
+
const perFlow = new Map();
|
|
944
|
+
for (const r of runs) perFlow.set(r.flow_name, [...(perFlow.get(r.flow_name) ?? []), r]);
|
|
945
|
+
runs = [...perFlow.values()].flatMap((rs) => rs.slice(-keep));
|
|
946
|
+
}
|
|
947
|
+
const report = metrics.hpi({ flows: runs, baselines: baseline.readBaselines() });
|
|
948
|
+
if (flags.out) store.writeAtomic(String(flags.out), `${JSON.stringify(report, null, 2)}\n`);
|
|
949
|
+
if (!runs.length) {
|
|
950
|
+
emit(flags, report, [
|
|
951
|
+
`no runs of any suite flow on this device yet (${all.length} unnamed run${all.length === 1 ? '' : 's'} in the log)`,
|
|
952
|
+
'run the agent side: node scripts/bench-hpi.mjs --device=' + dev.udid,
|
|
953
|
+
]);
|
|
954
|
+
return;
|
|
955
|
+
}
|
|
956
|
+
emit(flags, report, [
|
|
957
|
+
flags.last ? `the last ${num(flags.last)} run(s) of each flow, of ${all.filter((f) => f.flow_name).length} named runs in the log` : null,
|
|
958
|
+
'flow runs agent p50 human p50 HPI_time step_ratio turns esc',
|
|
959
|
+
...report.flows.map((f) =>
|
|
960
|
+
`${f.flow.padEnd(24)} ${String(f.runs).padStart(5)} ${`${f.agent_ms.p50}ms`.padStart(9)} ` +
|
|
961
|
+
`${(f.human_median_ms ? `${f.human_median_ms}ms` : '—').padStart(9)} ` +
|
|
962
|
+
`${(f.hpi_time ?? '—').toString().padStart(8)} ${(f.step_ratio ?? '—').toString().padStart(10)} ` +
|
|
963
|
+
`${(f.model_turns ?? '—').toString().padStart(5)} ${String(f.escalations).padStart(3)}`),
|
|
964
|
+
'',
|
|
965
|
+
`HPI_accuracy ${report.overall.hpi_accuracy} (${report.overall.runs} runs, ` +
|
|
966
|
+
`${report.overall.runs - runs.filter((r) => r.completed && !r.wrong_action_taken).length} not clean)`,
|
|
967
|
+
report.overall.hpi_time == null
|
|
968
|
+
? `HPI_time and HPI need a human baseline — none of ${report.overall.flows_measured} measured flow(s) has one yet.`
|
|
969
|
+
: `HPI_time ${report.overall.hpi_time} (harmonic mean over ${report.overall.flows_with_human_baseline} flow(s)), HPI ${report.overall.hpi}`,
|
|
970
|
+
`step_ratio ${report.overall.step_ratio ?? '—'} (target ≤1.5), model turns per flow ${report.overall.model_turns_median ?? '—'}`,
|
|
971
|
+
flags.out ? `wrote ${flags.out}` : null,
|
|
972
|
+
]);
|
|
973
|
+
return;
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
case 'escalations': {
|
|
977
|
+
const dev = await resolveDevice(flags.device);
|
|
978
|
+
const records = metrics.readEscalations(dev.udid, { limit: flags.last ? num(flags.last) : undefined });
|
|
979
|
+
const b = metrics.breakdown(records, {
|
|
980
|
+
session: flags.session === true ? metrics.sessionId() : (flags.session ? String(flags.session) : null),
|
|
981
|
+
flow: flags.flow ? String(flags.flow) : null,
|
|
982
|
+
});
|
|
983
|
+
if (flags.out) store.writeAtomic(String(flags.out), `${JSON.stringify(b, null, 2)}\n`);
|
|
984
|
+
emit(flags, b, [
|
|
985
|
+
`${b.total} escalation${b.total === 1 ? '' : 's'} on ${dev.name}`,
|
|
986
|
+
...metrics.REASONS
|
|
987
|
+
.filter((r) => b.by_reason[r])
|
|
988
|
+
.sort((a, c) => b.by_reason[c] - b.by_reason[a])
|
|
989
|
+
.map((r) => ` ${r.padEnd(20)} ${String(b.by_reason[r]).padStart(4)} `
|
|
990
|
+
+ (metrics.BUILT_FACULTIES.has(metrics.FACULTY[r])
|
|
991
|
+
? `not removed by: ${metrics.FACULTY[r]} [built]`
|
|
992
|
+
: `would be removed by: ${metrics.FACULTY[r]}`)),
|
|
993
|
+
b.total ? '' : null,
|
|
994
|
+
b.total ? `avoidable ${b.avoidable}/${b.total} (${b.avoidable_escalation_rate})` : null,
|
|
995
|
+
// Said out loud rather than left for someone to discover: the rate is
|
|
996
|
+
// 1.0 while no faculty exists, so the breakdown above is the number
|
|
997
|
+
// that decides the next phase.
|
|
998
|
+
b.total && b.avoidable_escalation_rate === 1
|
|
999
|
+
? (metrics.REASONS.some((r) => b.by_reason[r] && metrics.BUILT_FACULTIES.has(metrics.FACULTY[r]))
|
|
1000
|
+
? ' the rate is 1.0 because nothing resolves locally yet. A reason marked [built] is not a queue waiting on a phase — it is evidence the phase that shipped is not sufficient.'
|
|
1001
|
+
: ' every reason maps to a faculty that is not built yet, so this rate is 1.0 by construction. The per-reason counts are the steering wheel.')
|
|
1002
|
+
: null,
|
|
1003
|
+
b.total ? `model turns spent on escalations: ${b.model_turns_spent}` : null,
|
|
1004
|
+
// The log is per-device and shared. Said before the counts are used,
|
|
1005
|
+
// not after: two agents on one booted simulator write one interleaved
|
|
1006
|
+
// file, and CLAUDE.md makes these counts the thing that picks the next
|
|
1007
|
+
// faculty. A pooled breakdown errs toward whichever session made more
|
|
1008
|
+
// mistakes, which is a different question.
|
|
1009
|
+
b.pooled
|
|
1010
|
+
? 'WARNING these counts may pool more than one agent\'s work: '
|
|
1011
|
+
+ [
|
|
1012
|
+
b.session_count > 1 ? `${b.session_count} sessions` : null,
|
|
1013
|
+
b.unattributed ? `${b.unattributed} record(s) written before sessions were logged` : null,
|
|
1014
|
+
].filter(Boolean).join(', ')
|
|
1015
|
+
+ '. Narrow with --session (this process), --session=<id>, or --flow=<name>.'
|
|
1016
|
+
: null,
|
|
1017
|
+
b.session_count > 1 ? 'sessions:' : null,
|
|
1018
|
+
...(b.session_count > 1
|
|
1019
|
+
? b.sessions.map((x) => ` ${x.session_id.padEnd(22)} ${String(x.count).padStart(4)} ${x.client}`)
|
|
1020
|
+
: []),
|
|
1021
|
+
Object.keys(b.by_flow).length > 1 ? 'flows:' : null,
|
|
1022
|
+
...(Object.keys(b.by_flow).length > 1
|
|
1023
|
+
? Object.entries(b.by_flow).slice(0, 10).map(([n, c]) => ` ${n.padEnd(28)} ${String(c).padStart(4)}`)
|
|
1024
|
+
: []),
|
|
1025
|
+
b.top_screens.length ? 'top screens:' : null,
|
|
1026
|
+
...b.top_screens.map((s) => ` ${s.fingerprint.slice(0, 16).padEnd(18)} ${s.count}`),
|
|
1027
|
+
metrics.writeError() ? `WARNING a log write failed: ${metrics.writeError()}` : null,
|
|
1028
|
+
]);
|
|
1029
|
+
return;
|
|
1030
|
+
}
|
|
1031
|
+
|
|
727
1032
|
case 'devices': {
|
|
728
1033
|
const all = await listDevices();
|
|
729
1034
|
const shown = flags.all ? all : all.filter((d) => d.state === 'Booted');
|
|
@@ -772,6 +1077,52 @@ async function main() {
|
|
|
772
1077
|
* removed, and a fresh machine without it has not degraded from anything.
|
|
773
1078
|
* Strict fails on warn and fail, never on optional.
|
|
774
1079
|
*/
|
|
1080
|
+
/**
|
|
1081
|
+
* Is the device's display black, or is it only simframe that cannot read it?
|
|
1082
|
+
*
|
|
1083
|
+
* Two very different faults with one symptom, and telling them apart by hand
|
|
1084
|
+
* took an hour: `simctl io screenshot` on a wedged device wrote a valid PNG
|
|
1085
|
+
* whose 3.16 million pixels were all black, in 16.2 s. So the display pipeline
|
|
1086
|
+
* had failed and simframe's read was an accurate report of it.
|
|
1087
|
+
*
|
|
1088
|
+
* Slow on purpose-built-in: this runs once, only when capture has already
|
|
1089
|
+
* declared itself stalled, and 16 s of certainty beats an hour of guessing.
|
|
1090
|
+
*/
|
|
1091
|
+
async function blackScreenProbe(udid) {
|
|
1092
|
+
const file = path.join(os.tmpdir(), `simframe-probe-${Date.now()}.png`);
|
|
1093
|
+
const startedAt = Date.now();
|
|
1094
|
+
try {
|
|
1095
|
+
await screenshot(udid, file);
|
|
1096
|
+
const png = decodePng(fs.readFileSync(file));
|
|
1097
|
+
let lit = 0;
|
|
1098
|
+
for (let i = 0; i < png.data.length; i += 4) {
|
|
1099
|
+
if (png.data[i] > 12 || png.data[i + 1] > 12 || png.data[i + 2] > 12) lit += 1;
|
|
1100
|
+
}
|
|
1101
|
+
const ms = Date.now() - startedAt;
|
|
1102
|
+
const pixels = png.data.length / 4;
|
|
1103
|
+
if (lit === 0) {
|
|
1104
|
+
return {
|
|
1105
|
+
value: 'black',
|
|
1106
|
+
detail: `the device's display is rendering black — every one of ${pixels.toLocaleString()} pixels, `
|
|
1107
|
+
+ `confirmed through Apple's own screenshot path in ${ms}ms. This is the simulator, not simframe: `
|
|
1108
|
+
+ 'it often recovers on its own, and restarting the device also cures it. Re-resolving the '
|
|
1109
|
+
+ 'display port and rebinding the device have both been tried — 223 and 6 times — and neither '
|
|
1110
|
+
+ 'makes any difference, because there is nothing wrong with the handle.',
|
|
1111
|
+
};
|
|
1112
|
+
}
|
|
1113
|
+
return {
|
|
1114
|
+
value: 'readable-by-simctl',
|
|
1115
|
+
detail: `simctl can see ${lit.toLocaleString()} lit pixels of ${pixels.toLocaleString()} in ${ms}ms `
|
|
1116
|
+
+ 'while the daemon cannot read the surface at all. That is a simframe bug, not a wedged simulator — '
|
|
1117
|
+
+ 'worth reporting with this line.',
|
|
1118
|
+
};
|
|
1119
|
+
} catch (err) {
|
|
1120
|
+
return { value: 'unreadable', detail: `even simctl could not screenshot this device: ${err.message}` };
|
|
1121
|
+
} finally {
|
|
1122
|
+
fs.rmSync(file, { force: true });
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
|
|
775
1126
|
async function doctor({ json = false, strict = false, device } = {}) {
|
|
776
1127
|
const checks = [];
|
|
777
1128
|
// `level` is 'ok' | 'warn' | 'fail'. A warn means it works but not the way it
|
|
@@ -905,6 +1256,49 @@ async function doctor({ json = false, strict = false, device } = {}) {
|
|
|
905
1256
|
driver.available ? `${driver.name}: ${driver.version}` : driver.reason,
|
|
906
1257
|
{ key: 'input.driver', value: driver.available ? driver.name : null });
|
|
907
1258
|
}
|
|
1259
|
+
// When capture is wedged, say whose fault it is.
|
|
1260
|
+
//
|
|
1261
|
+
// Established the hard way: on a wedged device, Apple's own
|
|
1262
|
+
// `simctl io screenshot` still succeeds — and returns an image with zero
|
|
1263
|
+
// non-black pixels, in 16 seconds instead of one. The simulator's
|
|
1264
|
+
// display pipeline is rendering black; simframe's IOSurface read is not
|
|
1265
|
+
// the thing that broke. Re-resolving the port does not help, and neither
|
|
1266
|
+
// does rebinding the device: both were tried, the second six times.
|
|
1267
|
+
//
|
|
1268
|
+
// That distinction is the whole value of this check. "simframe cannot
|
|
1269
|
+
// read the display" invites someone to debug simframe; "the device's
|
|
1270
|
+
// display is black and Apple's screenshot agrees" tells them to restart
|
|
1271
|
+
// the device. The probe costs one screenshot and only runs when capture
|
|
1272
|
+
// has already given up.
|
|
1273
|
+
const wedged = store.captureHealth(d.udid)?.stalled;
|
|
1274
|
+
if (wedged) {
|
|
1275
|
+
const probe = await blackScreenProbe(d.udid);
|
|
1276
|
+
add(`display (${d.name})`, 'fail', probe.detail, { key: 'display.probe', value: probe.value });
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
// Reported next to the driver it is about. A driver that is present and
|
|
1280
|
+
// working is still useless if it holds a session for a device session
|
|
1281
|
+
// that no longer exists, and that state was invisible: taps were
|
|
1282
|
+
// dispatched successfully and moved nothing, five runs in a row.
|
|
1283
|
+
const session = await input.sessionHealth(d.udid);
|
|
1284
|
+
if (session.stale) {
|
|
1285
|
+
// The remedy used to read "the next action rebuilds it automatically;
|
|
1286
|
+
// simframe stop && simframe start does it now", and both halves were
|
|
1287
|
+
// wrong. The first was false wherever it mattered, because the rebuild
|
|
1288
|
+
// check was gated once per process and the MCP server is one process
|
|
1289
|
+
// for a whole session. The second names a command that fails twice:
|
|
1290
|
+
// `stop` needs `--device` when two simulators are booted, and then
|
|
1291
|
+
// refuses because a client holds the daemon, so the sequence that
|
|
1292
|
+
// actually works is `stop --device <udid> --force && start --device
|
|
1293
|
+
// <udid>` — a daemon restart, to fix a session, when rebuilding the
|
|
1294
|
+
// session is a thing the daemon can already do on request. It just had
|
|
1295
|
+
// no way in from outside. It does now.
|
|
1296
|
+
add(`input session (${d.name})`, 'warn', `stale — ${session.reason}. The next action rebuilds it; simframe input reset --device ${d.udid} does it now`,
|
|
1297
|
+
{ key: 'input.session', value: 'stale' });
|
|
1298
|
+
} else if (caps.input.supported) {
|
|
1299
|
+
add(`input session (${d.name})`, 'ok', session.reason ?? 'current with this device session',
|
|
1300
|
+
{ key: 'input.session', value: 'current' });
|
|
1301
|
+
}
|
|
908
1302
|
add(`text recognition (${d.name})`, 'ok',
|
|
909
1303
|
daemon ? 'simframed (in-process, off the framebuffer)' : 'sips + helper binary');
|
|
910
1304
|
if (!caps.ax.supported) {
|
|
@@ -923,8 +1317,20 @@ async function doctor({ json = false, strict = false, device } = {}) {
|
|
|
923
1317
|
if (probed.length) {
|
|
924
1318
|
const t0 = Date.now();
|
|
925
1319
|
const res = await api.getFrame(probed[0].udid);
|
|
926
|
-
|
|
927
|
-
|
|
1320
|
+
// The wedge's whole signature is that everything here reports success.
|
|
1321
|
+
// A black frame is 32 integer comparisons on a signature already
|
|
1322
|
+
// computed, and it is what separates "captured a frame" from "captured
|
|
1323
|
+
// a frame of a display that has stopped rendering".
|
|
1324
|
+
const dark = analyze.isBlackFrame(
|
|
1325
|
+
(res.state.history ?? []).find((h) => h.seq === res.state.seq)?.sig ?? null,
|
|
1326
|
+
);
|
|
1327
|
+
add('capture', dark ? 'warn' : 'ok',
|
|
1328
|
+
`frame #${res.state.seq} ${res.width}x${res.height} in ${Date.now() - t0}ms (age ${res.ageMs}ms)`
|
|
1329
|
+
+ (dark
|
|
1330
|
+
? ' — and every pixel of it is black. If the device is not showing a black screen on purpose,'
|
|
1331
|
+
+ ' this is the display pipeline having stopped rendering; it usually recovers on its own,'
|
|
1332
|
+
+ ` and ${'xcrun simctl shutdown'} / boot is the cure that always works.`
|
|
1333
|
+
: ''),
|
|
928
1334
|
{ key: 'capture.frames', value: res.state.seq });
|
|
929
1335
|
// A wedged device produces the same nothing as a quiet one, so doctor has
|
|
930
1336
|
// to ask the capture loop rather than look at the frames. `fail`, not
|
package/src/daemon.js
CHANGED
|
@@ -92,6 +92,15 @@ export async function runDaemon(device, options = {}) {
|
|
|
92
92
|
let prevSignature = null;
|
|
93
93
|
/** Recent frames, so a caller can diff against whatever it last saw rather
|
|
94
94
|
* than only against the frame that happened to precede this one. */
|
|
95
|
+
// A new capture session inherits no stall.
|
|
96
|
+
//
|
|
97
|
+
// capture-health.json is cleared when a stalled loop captures a frame again,
|
|
98
|
+
// and a loop that dies while stalled never gets to. So a fresh daemon ran
|
|
99
|
+
// healthily while `doctor` reported "stalled for 221s, 60 re-attaches" from
|
|
100
|
+
// its predecessor, and the display probe — correctly, on that input — called
|
|
101
|
+
// it a simframe bug. It was: this one.
|
|
102
|
+
store.writeCaptureHealth(udid, null);
|
|
103
|
+
|
|
95
104
|
let history = [];
|
|
96
105
|
/** seq + timestamp for every frame still on disk, so retention can be thinned by age. */
|
|
97
106
|
let ringIndex = [];
|
package/src/fingerprint.js
CHANGED
|
@@ -24,8 +24,14 @@ import * as regions from './regions.js';
|
|
|
24
24
|
* browser's address bar put "== example.com" into a screen's identity, so
|
|
25
25
|
* a different page read as a different screen, and OCR's ":" and "+" read
|
|
26
26
|
* off icons were identities of their own.
|
|
27
|
+
* 4 — not a token-rule change at all, and bumped anyway: the screen map now
|
|
28
|
+
* merges an OCR reading that sits inside a labelled ax element into that
|
|
29
|
+
* element, so the target list these rules run over is shorter on every
|
|
30
|
+
* list screen. Same rules, different input, therefore different hashes —
|
|
31
|
+
* and a stored hash that can never match again is the quietest kind of
|
|
32
|
+
* wrong, which is what this counter exists to prevent.
|
|
27
33
|
*/
|
|
28
|
-
export const TOKEN_RULES_VERSION =
|
|
34
|
+
export const TOKEN_RULES_VERSION = 4;
|
|
29
35
|
|
|
30
36
|
/** Frames are quantised to this, so sub-pixel drift and a nudged row do not matter. */
|
|
31
37
|
export const GRID = 24;
|