simframe 0.12.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +163 -9
- package/native/supervise.swift +63 -4
- package/package.json +1 -1
- package/scripts/article-md.mjs +185 -0
- package/scripts/ci-integration-local.sh +22 -1
- package/scripts/ci-memory.mjs +98 -15
- package/scripts/collect-rulings.mjs +14 -0
- package/scripts/eval-fingerprint.mjs +48 -3
- package/scripts/replay-rulings.mjs +206 -0
- package/scripts/score-rulings.mjs +19 -1
- package/src/actions.js +158 -18
- package/src/analyze.js +56 -0
- package/src/cli.js +215 -18
- package/src/fingerprint.js +10 -1
- package/src/index.js +337 -12
- package/src/input.js +4 -0
- package/src/mcp.js +7 -1
- package/src/metrics.js +68 -7
- package/src/navigate.js +28 -2
- package/src/ollama.js +232 -0
- package/src/platform/ios.js +30 -5
- package/src/refs.js +12 -1
- package/src/regions.js +54 -0
- package/src/screenmap.js +161 -6
- package/src/store.js +53 -0
- package/src/supervisor.js +108 -5
- package/src/view.js +84 -9
package/src/index.js
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
isBlackFrame,
|
|
14
14
|
maxCellDelta,
|
|
15
15
|
CELL_CHANGE,
|
|
16
|
+
describeMotion,
|
|
16
17
|
regionDeltas,
|
|
17
18
|
regionMap,
|
|
18
19
|
signatureDiff,
|
|
@@ -110,6 +111,48 @@ async function deviceGeometry(udid, state) {
|
|
|
110
111
|
};
|
|
111
112
|
}
|
|
112
113
|
|
|
114
|
+
/**
|
|
115
|
+
* How long to wait for a daemon to exist at all. A first build is slower than a
|
|
116
|
+
* spawn, which is what this number is sized for.
|
|
117
|
+
*/
|
|
118
|
+
export const READY_TIMEOUT_MS = 20_000;
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* How long to keep waiting once the daemon is demonstrably alive.
|
|
122
|
+
*
|
|
123
|
+
* Sized from the measurement that caused it: a hosted runner's simulator took
|
|
124
|
+
* roughly 27 s to produce its first frame while the daemon reported a healthy
|
|
125
|
+
* 75 ms median. Three times the ordinary budget, and still bounded.
|
|
126
|
+
*/
|
|
127
|
+
export const LIVE_DAEMON_CAP_MS = 60_000;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* How many gestures may land on a motionless screen before that is evidence.
|
|
131
|
+
*
|
|
132
|
+
* **This replaces a duration threshold, and the replacement is the point.** The
|
|
133
|
+
* first version asked "has the screen been still for a long time?" and needed
|
|
134
|
+
* 20 s before it would speak. A field report then caught a frame roughly three
|
|
135
|
+
* hours stale, presented as 130 ms old, on a screen still for **8.2 seconds** —
|
|
136
|
+
* so the check sat silently under its own gate during exactly the failure it
|
|
137
|
+
* was written for. A threshold chosen from one earlier example is not a
|
|
138
|
+
* mechanism.
|
|
139
|
+
*
|
|
140
|
+
* What says "dead surface" is not how long the screen has been quiet. It is
|
|
141
|
+
* that we kept touching it and *nothing moved at all*. One tap that changes
|
|
142
|
+
* nothing is ordinary — a disabled control, a form refusing a submit. Three in
|
|
143
|
+
* a row, with not one pixel of response, is a different claim.
|
|
144
|
+
*/
|
|
145
|
+
export const SURFACE_SUSPECT_INPUTS = 3;
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Slack between "input landed" and "the screen should have moved".
|
|
149
|
+
*
|
|
150
|
+
* A gesture only counts against the surface if it landed *inside* the quiet
|
|
151
|
+
* period with room to spare — otherwise the tap that ends the quiet period
|
|
152
|
+
* counts as evidence against it.
|
|
153
|
+
*/
|
|
154
|
+
export const SURFACE_MARGIN_MS = 400;
|
|
155
|
+
|
|
113
156
|
export function daemonStatus(udid) {
|
|
114
157
|
const meta = store.readJson(store.paths(udid).meta);
|
|
115
158
|
const pid = meta?.pid ?? null;
|
|
@@ -134,10 +177,13 @@ export async function ensureDaemon(deviceQuery, options = {}) {
|
|
|
134
177
|
// A dead daemon leaves its last state.json behind. Anything captured before
|
|
135
178
|
// we (re)started the loop is not evidence of a live screen, so ignore it.
|
|
136
179
|
const minCapturedAt = existing.alive ? 0 : Date.now();
|
|
180
|
+
// The pid we spawned, which is the only honest answer to "did a daemon come
|
|
181
|
+
// up" — see the comment on the wait below.
|
|
182
|
+
let spawnedPid = null;
|
|
137
183
|
if (!existing.alive) {
|
|
138
184
|
if (acquireSpawnLock(p.lock)) {
|
|
139
185
|
try {
|
|
140
|
-
await startEngine(device.udid, options);
|
|
186
|
+
spawnedPid = (await startEngine(device.udid, options)).pid;
|
|
141
187
|
} finally {
|
|
142
188
|
// Hold the lock briefly so a burst of callers does not double-spawn.
|
|
143
189
|
setTimeout(() => releaseSpawnLock(p.lock), 1500).unref?.();
|
|
@@ -146,16 +192,79 @@ export async function ensureDaemon(deviceQuery, options = {}) {
|
|
|
146
192
|
}
|
|
147
193
|
|
|
148
194
|
// The daemon may need a first build, which is slower than a spawn.
|
|
149
|
-
const deadline = Date.now() + (options.readyTimeoutMs ??
|
|
150
|
-
|
|
195
|
+
const deadline = Date.now() + (options.readyTimeoutMs ?? READY_TIMEOUT_MS);
|
|
196
|
+
// A live daemon that has not rendered yet is not a failed daemon.
|
|
197
|
+
//
|
|
198
|
+
// Measured on a hosted runner: `simframe start` gave up, and the daemon's own
|
|
199
|
+
// log — captured by the on-failure step eight seconds later — showed
|
|
200
|
+
// `frame=#1 age=1514ms, 1.0 fps, median 75.08ms`. It was working. The
|
|
201
|
+
// simulator's display had simply taken about 27 s to produce anything on a
|
|
202
|
+
// loaded build farm, against a 20 s budget measured on a developer's machine.
|
|
203
|
+
// Aborting there fails an entire run over a device that was about to work.
|
|
204
|
+
//
|
|
205
|
+
// So the budget applies to *getting a daemon*; once we have a live one, the
|
|
206
|
+
// wait extends to a hard cap. The cap still exists, because a daemon that is
|
|
207
|
+
// alive and never renders is a real failure and has to be reportable — but it
|
|
208
|
+
// is now a different sentence from a daemon that never started, and those two
|
|
209
|
+
// had read identically, which cost a log dive to tell apart.
|
|
210
|
+
const liveDeadline = Date.now() + LIVE_DAEMON_CAP_MS;
|
|
211
|
+
let sawLiveDaemon = false;
|
|
212
|
+
let sawProcess = false;
|
|
213
|
+
for (;;) {
|
|
151
214
|
const state = store.readJson(p.state);
|
|
152
215
|
if (state && state.capturedAt >= minCapturedAt && Date.now() - state.capturedAt < 30_000) {
|
|
153
216
|
return { device, state, started: !existing.alive };
|
|
154
217
|
}
|
|
218
|
+
// Two different questions, and for a long time one file answered both.
|
|
219
|
+
//
|
|
220
|
+
// `daemonStatus` reads meta.json, which the daemon writes in `claim()` —
|
|
221
|
+
// *after* `platform.attach`, its slowest startup step. So a daemon that has
|
|
222
|
+
// spawned and is attaching to CoreSimulator is indistinguishable, from
|
|
223
|
+
// here, from one that never started: both are `alive: false`. That is the
|
|
224
|
+
// condition the live cap below exists for, and it could never be reached in
|
|
225
|
+
// the case that produced it, because reaching it required the very file
|
|
226
|
+
// whose absence was the problem.
|
|
227
|
+
//
|
|
228
|
+
// Measured on a hosted runner, 2026-09-13: `simframe start` gave up after
|
|
229
|
+
// 20 s saying **"no daemon process came up"**, and the next step of the
|
|
230
|
+
// same job printed `● iPhone 16 Pro pid=9573 frame=#5 age=652ms` with a
|
|
231
|
+
// daemon log showing it had been capturing the whole time. The sentence was
|
|
232
|
+
// not merely unhelpful, it named the wrong condition — the fourth time this
|
|
233
|
+
// project has failed that way and the second time in two days.
|
|
234
|
+
//
|
|
235
|
+
// The pid we spawned is the honest signal and it was already in hand;
|
|
236
|
+
// `startEngine` was discarding it.
|
|
237
|
+
const claimed = daemonStatus(device.udid).alive;
|
|
238
|
+
const running = claimed || (spawnedPid != null && store.isProcessAlive(spawnedPid));
|
|
239
|
+
sawLiveDaemon = sawLiveDaemon || claimed;
|
|
240
|
+
sawProcess = sawProcess || running;
|
|
241
|
+
if (Date.now() >= (running ? liveDeadline : deadline)) break;
|
|
155
242
|
await sleep(80);
|
|
156
243
|
}
|
|
157
244
|
const tail = readLogTail(p.log);
|
|
158
|
-
throw new Error(
|
|
245
|
+
throw new Error(
|
|
246
|
+
`simframe daemon did not produce a frame for ${device.name} — `
|
|
247
|
+
+ `${readinessFailure({ sawLiveDaemon, sawProcess })}${tail ? `\n${tail}` : ''}`,
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Which of three failures this was.
|
|
253
|
+
*
|
|
254
|
+
* Extracted so it can be tested, for the reason `screenshotFailure` was: the
|
|
255
|
+
* whole defect here is a message naming the wrong condition, and a message is
|
|
256
|
+
* only checkable if something can ask for it without a device. Two of these
|
|
257
|
+
* three sentences did not exist until a runner produced each of them in turn and
|
|
258
|
+
* both arrived reading as the third.
|
|
259
|
+
*/
|
|
260
|
+
export function readinessFailure({ sawLiveDaemon, sawProcess }) {
|
|
261
|
+
const cap = Math.round(LIVE_DAEMON_CAP_MS / 1000);
|
|
262
|
+
if (sawLiveDaemon) return `the daemon is running and the display produced no frame in ${cap}s`;
|
|
263
|
+
if (sawProcess) {
|
|
264
|
+
return `the daemon process started but never claimed the device in ${cap}s`
|
|
265
|
+
+ ' — it is stuck attaching to the simulator rather than failing to launch';
|
|
266
|
+
}
|
|
267
|
+
return 'no daemon process came up';
|
|
159
268
|
}
|
|
160
269
|
|
|
161
270
|
/** Why the daemon was not used, when it was not. Surfaced by doctor. */
|
|
@@ -213,14 +322,16 @@ async function startEngine(udid, options) {
|
|
|
213
322
|
if (built.ok) {
|
|
214
323
|
engineFallbackReason = null;
|
|
215
324
|
recordFallback(udid, null);
|
|
216
|
-
|
|
217
|
-
|
|
325
|
+
// The pid, not just the name. Whether a daemon came up is a question
|
|
326
|
+
// about a process, and answering it from meta.json — which the daemon
|
|
327
|
+
// writes only after attaching — could not tell "never started" from
|
|
328
|
+
// "still starting". See the wait in `ensureDaemon`.
|
|
329
|
+
return { engine: 'simframed', pid: engine.spawnDaemon(udid, options)?.pid ?? null };
|
|
218
330
|
}
|
|
219
331
|
engineFallbackReason = built.reason ?? 'simframed unavailable';
|
|
220
332
|
recordFallback(udid, engineFallbackReason);
|
|
221
333
|
}
|
|
222
|
-
spawnNodeDaemon(udid, options);
|
|
223
|
-
return 'screenshot';
|
|
334
|
+
return { engine: 'screenshot', pid: spawnNodeDaemon(udid, options)?.pid ?? null };
|
|
224
335
|
}
|
|
225
336
|
|
|
226
337
|
function spawnNodeDaemon(udid, options) {
|
|
@@ -244,6 +355,7 @@ function spawnNodeDaemon(udid, options) {
|
|
|
244
355
|
env: process.env,
|
|
245
356
|
});
|
|
246
357
|
child.unref();
|
|
358
|
+
return child;
|
|
247
359
|
}
|
|
248
360
|
|
|
249
361
|
function acquireSpawnLock(lockFile) {
|
|
@@ -384,6 +496,49 @@ export function liveness(udid, state) {
|
|
|
384
496
|
if (!damageDriven && ageMs > STALE_FRAME_MS) {
|
|
385
497
|
return { ok: false, ageMs, stalled: false, note: `capture loop is stalled: newest frame is ${ageMs}ms old` };
|
|
386
498
|
}
|
|
499
|
+
|
|
500
|
+
// A live loop re-reading a dead surface, which every other signal calls fine.
|
|
501
|
+
//
|
|
502
|
+
// Reported from the field on 0.12.2, and it is the most expensive shape this
|
|
503
|
+
// tool has produced: `sim_look` served the *login screen* for three minutes
|
|
504
|
+
// while the app was four screens deep in a wizard, and announced the frame as
|
|
505
|
+
// 66ms old. The daemon was not lying — the frame really was new. It was a new
|
|
506
|
+
// read of a surface that had stopped updating. Every check above passes: the
|
|
507
|
+
// process is alive, no read failed, the age is tiny.
|
|
508
|
+
//
|
|
509
|
+
// The contradiction is between two numbers we already have. `stableForMs`
|
|
510
|
+
// says the screen has not changed in N ms; `lastInputAt` says we were
|
|
511
|
+
// delivering taps and swipes during that window. A screen that has not moved
|
|
512
|
+
// since *before* we last touched it, for long enough that several gestures
|
|
513
|
+
// landed inside the quiet period, is not a calm screen. The tester's own
|
|
514
|
+
// figures: stable for 159,753 ms across three screen transitions.
|
|
515
|
+
//
|
|
516
|
+
// Deliberately a WARNING and not a failure. A genuinely inert screen is
|
|
517
|
+
// possible — a form that rejects every tap, a disabled control tapped twice —
|
|
518
|
+
// and turning that into a hard error would be a confident wrong answer of
|
|
519
|
+
// exactly the kind this project keeps paying for. It says what it sees and
|
|
520
|
+
// names the arbiter.
|
|
521
|
+
const stableForMs = state?.stableForMs;
|
|
522
|
+
if (Number.isFinite(stableForMs)) {
|
|
523
|
+
// The moment the screen last moved. Every gesture delivered after it landed
|
|
524
|
+
// on a screen that did not react.
|
|
525
|
+
const lastChange = Date.now() - stableForMs + SURFACE_MARGIN_MS;
|
|
526
|
+
const ignored = store.inputTimes(udid).filter((t) => t > lastChange);
|
|
527
|
+
if (ignored.length >= SURFACE_SUSPECT_INPUTS) {
|
|
528
|
+
const quiet = Math.round(stableForMs / 1000);
|
|
529
|
+
return {
|
|
530
|
+
ok: true,
|
|
531
|
+
ageMs,
|
|
532
|
+
stalled: false,
|
|
533
|
+
suspectSurface: true,
|
|
534
|
+
ignoredInputs: ignored.length,
|
|
535
|
+
note: `${ignored.length} gestures have been delivered without the screen moving at all `
|
|
536
|
+
+ `(still for ${quiet < 1 ? `${Math.round(stableForMs)}ms` : `${quiet}s`}), `
|
|
537
|
+
+ 'so the frames are new and the surface behind them may be dead. '
|
|
538
|
+
+ 'The accessibility tree is read separately and is the tiebreaker; `simframe revive` re-attaches capture.',
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
}
|
|
387
542
|
return { ok: true, ageMs, stalled: false, note: null };
|
|
388
543
|
}
|
|
389
544
|
|
|
@@ -709,6 +864,37 @@ function timingOfNow(udid, state) {
|
|
|
709
864
|
* sampled after the fact is the single most common way to wait for a change
|
|
710
865
|
* that has already happened.
|
|
711
866
|
*/
|
|
867
|
+
/**
|
|
868
|
+
* How far back "still moving" looks, in milliseconds.
|
|
869
|
+
*
|
|
870
|
+
* **In time, not in frames**, and that distinction is the whole of it. The
|
|
871
|
+
* first version kept the last eight frame pairs, on the reasoning that at a
|
|
872
|
+
* 60 ms poll floor eight frames is about half a second. Frames are not a clock:
|
|
873
|
+
* capture is damage-driven with a slow idle floor, so on a quiet screen eight
|
|
874
|
+
* frames spanned *sixteen seconds* — and the window still held the transition
|
|
875
|
+
* that had brought us to the screen. A home screen with one animated widget
|
|
876
|
+
* duly reported movement in all thirty-two regions.
|
|
877
|
+
*
|
|
878
|
+
* One second, long enough that a stepping animation is not missed between two
|
|
879
|
+
* frames and short enough that the entry transition is gone by the time a
|
|
880
|
+
* settle gives up.
|
|
881
|
+
*/
|
|
882
|
+
export const MOTION_WINDOW_MS = 1000;
|
|
883
|
+
|
|
884
|
+
/**
|
|
885
|
+
* How young a frame has to be to count as "the screen right now".
|
|
886
|
+
*
|
|
887
|
+
* The daemon's idle floor is 2,000 ms — on a screen with no damage it captures
|
|
888
|
+
* anyway, at that cadence, precisely so state stays fresh enough to reason
|
|
889
|
+
* about. So a frame younger than that floor plus a margin is the current
|
|
890
|
+
* screen; older than it means the loop has missed its own beat, which is what
|
|
891
|
+
* `liveness` is for.
|
|
892
|
+
*
|
|
893
|
+
* Tied to the daemon's floor rather than chosen, because a number chosen here
|
|
894
|
+
* would drift away from it silently the first time the floor moved.
|
|
895
|
+
*/
|
|
896
|
+
export const FRAME_IS_CURRENT_MS = 2500;
|
|
897
|
+
|
|
712
898
|
export async function waitFor(
|
|
713
899
|
deviceQuery,
|
|
714
900
|
{ mode = 'settle', since, stableMs = 600, timeoutMs = 8000, reactionMs = 2500, baselineHash, options } = {},
|
|
@@ -750,6 +936,29 @@ export async function waitFor(
|
|
|
750
936
|
? hexToSignature(resolved.entry.sig)
|
|
751
937
|
: (requested == null ? hexToSignature(currentSig(first) ?? '') : null);
|
|
752
938
|
let smallChange = false;
|
|
939
|
+
/**
|
|
940
|
+
* Where the screen moved while we waited — item 123.
|
|
941
|
+
*
|
|
942
|
+
* The per-region maximum over a trailing window of frame pairs, so a settle
|
|
943
|
+
* that gives up can say *where* rather than only that it did.
|
|
944
|
+
*
|
|
945
|
+
* A window and not a running max over the whole wait, which is what the first
|
|
946
|
+
* version did and it was useless: every wait begins with the transition that
|
|
947
|
+
* brought us here, so a home screen with one animated widget reported
|
|
948
|
+
* movement in all thirty-two regions and "spread across the screen". The
|
|
949
|
+
* question a settle failure asks is *what is still moving*, not what has
|
|
950
|
+
* moved at any point since we started looking.
|
|
951
|
+
*
|
|
952
|
+
* A window rather than the last pair alone, because a blinking or stepping
|
|
953
|
+
* animation is between frames as often as not, and a single pair would
|
|
954
|
+
* report a still screen that is not one.
|
|
955
|
+
*
|
|
956
|
+
* Costs one subtraction per region per frame over a signature that was
|
|
957
|
+
* already read and parsed for `smallChange`.
|
|
958
|
+
*/
|
|
959
|
+
const motionRing = [];
|
|
960
|
+
let motionPrev = null;
|
|
961
|
+
let motionSeq = null;
|
|
753
962
|
/** Frames the display was not rendering at all. See `isBlackFrame`. */
|
|
754
963
|
let blackFrames = 0;
|
|
755
964
|
let blackSinceStart = null;
|
|
@@ -795,6 +1004,46 @@ export async function waitFor(
|
|
|
795
1004
|
sawChange = false;
|
|
796
1005
|
}
|
|
797
1006
|
|
|
1007
|
+
/**
|
|
1008
|
+
* What the daemon says is animating, if anything — and it has always known.
|
|
1009
|
+
*
|
|
1010
|
+
* `Motion.state` in the daemon localises a small persistent animation and
|
|
1011
|
+
* publishes it as `motion.animating`, a bounding box, alongside its own
|
|
1012
|
+
* `settled` flag. Nothing in JavaScript read either. `waitFor` decided
|
|
1013
|
+
* stillness from `stableForMs` alone, which is derived from a *mean* over the
|
|
1014
|
+
* grid, and a spinner does not move a mean.
|
|
1015
|
+
*
|
|
1016
|
+
* Measured on the testbed's Diagnostics screen, which now carries one on
|
|
1017
|
+
* purpose: frames arriving every 85 ms, `motion.animating` boxed at 13x13,
|
|
1018
|
+
* `motion.settled` **false** — and `stableForMs` **79,207**. Seventy-nine
|
|
1019
|
+
* seconds of claimed stillness on a screen that had never once stopped. A
|
|
1020
|
+
* field report had already called this in as `settled after 62ms` on a
|
|
1021
|
+
* still-loading screen.
|
|
1022
|
+
*
|
|
1023
|
+
* Deliberately reported and **not** acted on. Requiring the daemon's `settled`
|
|
1024
|
+
* here would be correct for a spinner and wrong for a text caret, which is
|
|
1025
|
+
* also small, also persistent, and must never stop a screen from settling —
|
|
1026
|
+
* and choosing between them needs a caret measured, not a threshold picked
|
|
1027
|
+
* today. So the disagreement becomes visible instead of being resolved by
|
|
1028
|
+
* guess: the caller is told a part of the screen is still moving and where,
|
|
1029
|
+
* and can decide. The decision is an open item with a number attached to it.
|
|
1030
|
+
*/
|
|
1031
|
+
const animatingNow = () => {
|
|
1032
|
+
const box = last?.motion?.animating;
|
|
1033
|
+
if (!box) return null;
|
|
1034
|
+
return { ...box, daemonSettled: last.motion.settled === true };
|
|
1035
|
+
};
|
|
1036
|
+
|
|
1037
|
+
/** The last second of movement, collapsed to one per-region maximum. */
|
|
1038
|
+
function motionSummary() {
|
|
1039
|
+
const cutoff = Date.now() - MOTION_WINDOW_MS;
|
|
1040
|
+
const recent = motionRing.filter((e) => e.at >= cutoff);
|
|
1041
|
+
if (!recent.length) return null;
|
|
1042
|
+
const max = recent[0].deltas.map((_, i) => Math.max(...recent.map((e) => e.deltas[i] ?? 0)));
|
|
1043
|
+
if (!max.some((d) => d > 0)) return null;
|
|
1044
|
+
return { deltas: max, map: regionMap(max), windowMs: MOTION_WINDOW_MS, ...describeMotion(max) };
|
|
1045
|
+
}
|
|
1046
|
+
|
|
798
1047
|
const done = (satisfied, extra = {}) => ({
|
|
799
1048
|
device,
|
|
800
1049
|
state: last,
|
|
@@ -812,6 +1061,24 @@ export async function waitFor(
|
|
|
812
1061
|
// because the app drew black. What makes it the capture wedge is that it
|
|
813
1062
|
// stays black while input is being delivered, and the caller knows that.
|
|
814
1063
|
blackMs: blackSinceStart ? Date.now() - blackSinceStart : 0,
|
|
1064
|
+
// Only worth carrying when something actually moved; a still screen that
|
|
1065
|
+
// never satisfied the wait has nothing to point at.
|
|
1066
|
+
motion: motionSummary(),
|
|
1067
|
+
animating: animatingNow(),
|
|
1068
|
+
/**
|
|
1069
|
+
* Everything that decided this answer, for the failure message.
|
|
1070
|
+
*
|
|
1071
|
+
* Two CI cycles were spent guessing at a `screen did not settle within
|
|
1072
|
+
* 25008ms` — a sentence that names the one number which is never the
|
|
1073
|
+
* reason. The wait turns on three things and the log carried none of them:
|
|
1074
|
+
* how long the screen had actually been still, how much stillness was
|
|
1075
|
+
* being asked for, and whether any frame arrived at all. "Still for 1,200
|
|
1076
|
+
* of the 1,400 ms required" and "still for 135,000 ms and nothing arrived"
|
|
1077
|
+
* are opposite diagnoses and they had one sentence between them.
|
|
1078
|
+
*/
|
|
1079
|
+
stillForMs: Number.isFinite(last?.stableForMs) ? last.stableForMs : null,
|
|
1080
|
+
stableMsRequired: stableMs,
|
|
1081
|
+
framesSeen: Number.isFinite(last?.seq) ? last.seq - startSeq : null,
|
|
815
1082
|
baselineHash: baselineHashValue,
|
|
816
1083
|
baselineResolved,
|
|
817
1084
|
waitedMs: Date.now() - startedAt,
|
|
@@ -865,6 +1132,23 @@ export async function waitFor(
|
|
|
865
1132
|
}
|
|
866
1133
|
}
|
|
867
1134
|
|
|
1135
|
+
// Track where it is moving, on new frames only — comparing a frame with
|
|
1136
|
+
// itself is a row of zeros that would dilute nothing but waste the work.
|
|
1137
|
+
if (state.seq !== motionSeq) {
|
|
1138
|
+
const sigHex = currentSig(state);
|
|
1139
|
+
const sig = sigHex ? hexToSignature(sigHex) : null;
|
|
1140
|
+
if (sig) {
|
|
1141
|
+
if (motionPrev) {
|
|
1142
|
+
motionRing.push({ at: Date.now(), deltas: regionDeltas(sig, motionPrev) });
|
|
1143
|
+
// Bounded in time by the summary and in length here, so a screen
|
|
1144
|
+
// rendering at 60 fps for ten seconds cannot grow this without end.
|
|
1145
|
+
while (motionRing.length > 240) motionRing.shift();
|
|
1146
|
+
}
|
|
1147
|
+
motionPrev = sig;
|
|
1148
|
+
}
|
|
1149
|
+
motionSeq = state.seq;
|
|
1150
|
+
}
|
|
1151
|
+
|
|
868
1152
|
// A pause that turned out not to be the end of the transition. Only
|
|
869
1153
|
// pauses followed by more movement count: the quiet at the end of a
|
|
870
1154
|
// settle is the answer, not a gap.
|
|
@@ -900,10 +1184,30 @@ export async function waitFor(
|
|
|
900
1184
|
// the caller acted can never satisfy it; once the change is seen,
|
|
901
1185
|
// stableForMs is measured from that change. Plain "stable" has no such
|
|
902
1186
|
// requirement — an already-still screen genuinely is stable.
|
|
903
|
-
//
|
|
904
|
-
//
|
|
1187
|
+
//
|
|
1188
|
+
// The evidence must be *current*, and that used to be spelled "at least
|
|
1189
|
+
// one frame must arrive during the call". The intent is right and the
|
|
1190
|
+
// spelling made the wait unsatisfiable on exactly the screens it is
|
|
1191
|
+
// asked about most. Capture is damage-driven: a screen that is not
|
|
1192
|
+
// moving produces no new frame by design, so on a still screen the
|
|
1193
|
+
// condition asks for evidence the system has deliberately chosen not to
|
|
1194
|
+
// generate, and the wait burns its whole budget and reports *"screen did
|
|
1195
|
+
// not settle"* about a screen that has been motionless throughout.
|
|
1196
|
+
//
|
|
1197
|
+
// Measured here rather than argued: a flow step reported `did not settle
|
|
1198
|
+
// within 1547ms` while its own evidence line read **still for 3548ms of
|
|
1199
|
+
// the 1400ms required; NO frame arrived while waiting**. Three and a half
|
|
1200
|
+
// seconds of stillness, two and a half times what was asked, refused.
|
|
1201
|
+
//
|
|
1202
|
+
// So currency is a question about time, not about a counter — the same
|
|
1203
|
+
// correction the motion window needed. A frame younger than the capture
|
|
1204
|
+
// loop's own idle floor *is* the screen as it is now. `liveness` is
|
|
1205
|
+
// checked at the top of every iteration and already rejects a stale file
|
|
1206
|
+
// from a dead daemon, which is the case the counter was really guarding.
|
|
905
1207
|
const freshFrames = state.seq - startSeq;
|
|
906
|
-
|
|
1208
|
+
const age = Date.now() - (state.capturedAt ?? 0);
|
|
1209
|
+
const current = freshFrames >= 1 || age <= FRAME_IS_CURRENT_MS;
|
|
1210
|
+
if (sawChange && current && state.stableForMs >= stableMs) {
|
|
907
1211
|
return done(true);
|
|
908
1212
|
}
|
|
909
1213
|
}
|
|
@@ -1431,7 +1735,28 @@ async function locateWith(
|
|
|
1431
1735
|
const target = index != null ? candidates[index] : candidates[0];
|
|
1432
1736
|
if (!target) {
|
|
1433
1737
|
const visible = entry.targets.filter((t) => t.label);
|
|
1434
|
-
const
|
|
1738
|
+
const shown = visible.slice(0, 12).map((t) => t.label);
|
|
1739
|
+
// Say when the list is cut. A field report found `"Work Orders" is not on
|
|
1740
|
+
// this screen. Visible: …` on a screen whose own element map, three lines
|
|
1741
|
+
// below in the same reply, listed `#25 text 200,836 Work Orders` — it was
|
|
1742
|
+
// simply past the twelfth entry. A truncated list that does not announce
|
|
1743
|
+
// its truncation reads as an exhaustive one.
|
|
1744
|
+
const sample = shown.join(', ') + (visible.length > shown.length ? `, …and ${visible.length - shown.length} more` : '');
|
|
1745
|
+
// An index past the end is not an absent element, and reporting it as one
|
|
1746
|
+
// sent a tester looking for a control that was on the screen all along.
|
|
1747
|
+
// `{"tap": "Work Orders", "index": 1}` on a screen with exactly one match
|
|
1748
|
+
// said "not on this screen" while the map listed it.
|
|
1749
|
+
if (index != null && candidates.length) {
|
|
1750
|
+
throw metrics.tag(
|
|
1751
|
+
new Error(
|
|
1752
|
+
`"${query}" matches ${candidates.length} thing${candidates.length === 1 ? '' : 's'} on this screen,`
|
|
1753
|
+
+ ` so index ${index} is out of range — valid indices are 0..${candidates.length - 1}.`
|
|
1754
|
+
+ ` Nearest: ${candidates.slice(0, 4).map((c) => `[${candidates.indexOf(c)}] ${JSON.stringify(String(c.label ?? '').slice(0, 28))}`).join(', ')}`,
|
|
1755
|
+
),
|
|
1756
|
+
'ambiguous_intent',
|
|
1757
|
+
{ candidates: candidates.slice(0, 8), intent: query, ambiguous: true },
|
|
1758
|
+
);
|
|
1759
|
+
}
|
|
1435
1760
|
throw metrics.tag(
|
|
1436
1761
|
new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
|
|
1437
1762
|
from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
|
package/src/input.js
CHANGED
|
@@ -623,6 +623,10 @@ export function shouldRebuildSession({ stale, bootedAt }, rebuiltFor) {
|
|
|
623
623
|
const rebuiltForBoot = new Map();
|
|
624
624
|
export async function ensureFreshSession(udid) {
|
|
625
625
|
if (!udid) return null;
|
|
626
|
+
// Every input path calls this first, so it is the one place that knows the
|
|
627
|
+
// hands are about to move. `liveness` needs that timestamp to tell a calm
|
|
628
|
+
// screen from a dead surface — see `store.noteInput`.
|
|
629
|
+
store.noteInput(udid);
|
|
626
630
|
const health = await sessionHealth(udid);
|
|
627
631
|
if (!shouldRebuildSession(health, rebuiltForBoot.get(udid))) return null;
|
|
628
632
|
rebuiltForBoot.set(udid, health.bootedAt);
|
package/src/mcp.js
CHANGED
|
@@ -459,7 +459,13 @@ function header(device, state, ageMs, extra = '') {
|
|
|
459
459
|
}
|
|
460
460
|
|
|
461
461
|
function livenessLine(live) {
|
|
462
|
-
|
|
462
|
+
// `ok` is not the only thing worth saying. A surface that has stopped
|
|
463
|
+
// updating leaves every hard signal green — the loop is alive, no read
|
|
464
|
+
// failed, the frame is milliseconds old — and reporting nothing is how
|
|
465
|
+
// `sim_look` served a three-minute-old login screen while announcing it as
|
|
466
|
+
// 66ms old. A note that exists must reach the caller.
|
|
467
|
+
if (live?.note) return `WARNING: ${live.note}`;
|
|
468
|
+
return null;
|
|
463
469
|
}
|
|
464
470
|
|
|
465
471
|
function sinceLine(since) {
|
package/src/metrics.js
CHANGED
|
@@ -151,7 +151,7 @@ export const readSupervisions = (udid, opts) => readJsonl(metricPaths(udid).supe
|
|
|
151
151
|
*/
|
|
152
152
|
export function recordSupervision(udid, {
|
|
153
153
|
session, index, step, edge, screen, decision, from, reason, ms,
|
|
154
|
-
stillMs, p95, samples, expect, failure, outcome,
|
|
154
|
+
stillMs, p95, samples, expect, failure, outcome, supervisor, situation,
|
|
155
155
|
}) {
|
|
156
156
|
// Swallowed rather than thrown, unlike `recordEscalation`'s guard, and the
|
|
157
157
|
// difference is deliberate: this is called from inside a flow's failure
|
|
@@ -171,6 +171,19 @@ export function recordSupervision(udid, {
|
|
|
171
171
|
screen_fingerprint: screen ?? null,
|
|
172
172
|
decision,
|
|
173
173
|
from: from ?? 'model',
|
|
174
|
+
// *Which* judge, not just that there was one.
|
|
175
|
+
//
|
|
176
|
+
// Every arm of the capacity comparison writes to this one log, and without
|
|
177
|
+
// this field a population collected under Apple and one collected under a
|
|
178
|
+
// 14B are one undifferentiated file — the comparison the owner asked for
|
|
179
|
+
// would be unreadable from its own data. `from` says rule-or-model; this
|
|
180
|
+
// says which model.
|
|
181
|
+
supervisor: supervisor ?? null,
|
|
182
|
+
// The question, not only the answer. Without it, asking a second judge
|
|
183
|
+
// about the same situations means driving the device a second time — which
|
|
184
|
+
// puts the device's own variance inside a comparison that is about the
|
|
185
|
+
// judges. Null for a rule-sourced ruling, which never composed one.
|
|
186
|
+
situation: situation ?? null,
|
|
174
187
|
// Recorded, never presented as the ground for what happened: the supervisor
|
|
175
188
|
// has returned a correct decision with a reason citing a rule that did not
|
|
176
189
|
// apply. Keeping it is how that stays measurable instead of anecdotal.
|
|
@@ -287,11 +300,30 @@ const VERIFYING_STEPS = new Set([
|
|
|
287
300
|
*/
|
|
288
301
|
export function reasonForStepError(step, err) {
|
|
289
302
|
const tagged = escalationOf(err);
|
|
290
|
-
if (tagged) return tagged;
|
|
303
|
+
if (tagged) return { ...tagged, classified: true };
|
|
291
304
|
const action = step?.action;
|
|
292
|
-
if (action === 'confirm' || action === 'chooseAny')
|
|
293
|
-
|
|
294
|
-
|
|
305
|
+
if (action === 'confirm' || action === 'chooseAny') {
|
|
306
|
+
return { reason: 'novel_dialog', candidates: [], tried: [], classified: true };
|
|
307
|
+
}
|
|
308
|
+
// Everything else is a *fallback*, and it now says so.
|
|
309
|
+
//
|
|
310
|
+
// This had two branches that returned the same value, which made it look
|
|
311
|
+
// like it discriminated. It does not: any step that threw without a site
|
|
312
|
+
// tagging it lands here. In a real field session that was **90% of all
|
|
313
|
+
// escalations** — and `FACULTY` then reported every one of them as evidence
|
|
314
|
+
// against "sense of time (Phase 11)", a claim nothing in the record supports.
|
|
315
|
+
//
|
|
316
|
+
// The tester's own first call failed with `unknown step "wait_for"` — a typo
|
|
317
|
+
// — and that too would be filed as evidence about which faculty to build
|
|
318
|
+
// next. CLAUDE.md calls this log the steering wheel; a steering wheel that
|
|
319
|
+
// pools typos, inert controls and slow lists into one reason is pointing
|
|
320
|
+
// somewhere nobody chose.
|
|
321
|
+
//
|
|
322
|
+
// No sixth reason: "unknown is not a reason" stays, and a vocabulary that
|
|
323
|
+
// admits "other" collects a pile of "other". What changes is that the record
|
|
324
|
+
// carries whether the reason was *read off the failure* or *assumed*, and
|
|
325
|
+
// the report declines to recommend a faculty for the assumed ones.
|
|
326
|
+
return { reason: 'verification_failed', candidates: [], tried: [], classified: false };
|
|
295
327
|
}
|
|
296
328
|
|
|
297
329
|
/**
|
|
@@ -312,6 +344,11 @@ export const PLAN_REASONS = {
|
|
|
312
344
|
'no-route': 'no_plan',
|
|
313
345
|
'unreplayable-edge': 'no_plan',
|
|
314
346
|
'unknown-flow': 'no_plan',
|
|
347
|
+
// A route that ran and did not land. Not `no_plan`: there *was* a plan and it
|
|
348
|
+
// was followed — what could not be confirmed is that it worked, which is what
|
|
349
|
+
// `verification_failed` means everywhere else in this file.
|
|
350
|
+
'route-halted': 'verification_failed',
|
|
351
|
+
'arrived-elsewhere': 'verification_failed',
|
|
315
352
|
};
|
|
316
353
|
|
|
317
354
|
/** A short, bounded description of a candidate element, for the log. */
|
|
@@ -422,6 +459,10 @@ export function recordEscalation(udid, {
|
|
|
422
459
|
modelTurns = 1,
|
|
423
460
|
wallMs = null,
|
|
424
461
|
detail = null,
|
|
462
|
+
// Was this reason read off the failure, or assumed because nothing said?
|
|
463
|
+
// Default `false`, so a caller that does not think about it cannot
|
|
464
|
+
// accidentally claim precision it does not have.
|
|
465
|
+
classified = false,
|
|
425
466
|
} = {}) {
|
|
426
467
|
if (!REASONS.includes(reason)) throw new Error(`not an escalation reason: ${reason}`);
|
|
427
468
|
if (!OUTCOMES.includes(outcome)) throw new Error(`not an escalation outcome: ${outcome}`);
|
|
@@ -437,6 +478,7 @@ export function recordEscalation(udid, {
|
|
|
437
478
|
// What was asked for, in the caller's words. Ground truth for Phase 17's
|
|
438
479
|
// go/no-go, and on its own it answers "what kind of decision is costing us".
|
|
439
480
|
intent: intent ? String(intent).slice(0, 120) : null,
|
|
481
|
+
classified: Boolean(classified),
|
|
440
482
|
step_index: stepIndex,
|
|
441
483
|
screen_fingerprint: fingerprint,
|
|
442
484
|
reason,
|
|
@@ -617,7 +659,9 @@ export function hpi({ flows, baselines = {} }) {
|
|
|
617
659
|
*/
|
|
618
660
|
export function breakdown(records, { session = null, flow = null } = {}) {
|
|
619
661
|
const byReason = {};
|
|
620
|
-
|
|
662
|
+
const classifiedByReason = {};
|
|
663
|
+
const assumedByReason = {};
|
|
664
|
+
for (const r of REASONS) { byReason[r] = 0; classifiedByReason[r] = 0; assumedByReason[r] = 0; }
|
|
621
665
|
const byScreen = new Map();
|
|
622
666
|
const byOutcome = {};
|
|
623
667
|
const bySession = new Map();
|
|
@@ -641,6 +685,12 @@ export function breakdown(records, { session = null, flow = null } = {}) {
|
|
|
641
685
|
}
|
|
642
686
|
if (r.flow_name) byFlow.set(r.flow_name, (byFlow.get(r.flow_name) ?? 0) + 1);
|
|
643
687
|
byReason[r.reason] += 1;
|
|
688
|
+
// Three states, not two. A record written before this field existed makes
|
|
689
|
+
// no claim either way, and folding it in with "assumed" would make an old
|
|
690
|
+
// log look like a diagnosis failure — a warning that cries wolf is how a
|
|
691
|
+
// real one gets ignored, which this file already knows in another place.
|
|
692
|
+
if (r.classified === true) classifiedByReason[r.reason] += 1;
|
|
693
|
+
else if (r.classified === false) assumedByReason[r.reason] += 1;
|
|
644
694
|
byOutcome[r.outcome] = (byOutcome[r.outcome] ?? 0) + 1;
|
|
645
695
|
// Already avoided locally, so not avoidable by anything unbuilt.
|
|
646
696
|
if (r.outcome !== 'resolved_locally') avoidable += 1;
|
|
@@ -667,9 +717,20 @@ export function breakdown(records, { session = null, flow = null } = {}) {
|
|
|
667
717
|
pooled: sessions.length > 1 || unattributed > 0,
|
|
668
718
|
by_flow: Object.fromEntries([...byFlow.entries()].sort((a, b) => b[1] - a[1])),
|
|
669
719
|
by_reason: byReason,
|
|
720
|
+
// How many of each reason were *read off the failure* rather than assumed.
|
|
721
|
+
//
|
|
722
|
+
// The breakdown above picks the next phase, so its precision has to be
|
|
723
|
+
// visible in it. A reason that is mostly assumed is not a finding about an
|
|
724
|
+
// app; it is a count of things nothing could classify, and reading it as a
|
|
725
|
+
// verdict on a faculty is how the instrument came to disagree with a
|
|
726
|
+
// tester who was right.
|
|
727
|
+
classified_by_reason: classifiedByReason,
|
|
728
|
+
assumed_by_reason: assumedByReason,
|
|
670
729
|
by_outcome: byOutcome,
|
|
730
|
+
// Only where the reason was actually read. A faculty named against a pile
|
|
731
|
+
// of assumptions is advice with nothing behind it.
|
|
671
732
|
faculty: Object.fromEntries(
|
|
672
|
-
REASONS.filter((r) =>
|
|
733
|
+
REASONS.filter((r) => classifiedByReason[r]).map((r) => [r, `${FACULTY[r]}${BUILT_FACULTIES.has(FACULTY[r]) ? ' [built]' : ''}`]),
|
|
673
734
|
),
|
|
674
735
|
avoidable,
|
|
675
736
|
avoidable_escalation_rate: total ? Number((avoidable / total).toFixed(3)) : null,
|
package/src/navigate.js
CHANGED
|
@@ -96,14 +96,40 @@ export async function goto(deviceQuery, target, { options, ...runOptions } = {})
|
|
|
96
96
|
|
|
97
97
|
const result = await runScript(udid, { steps, stopOnUnexpected: true, ...runOptions });
|
|
98
98
|
const arrived = await api.screenIdentity(udid, {});
|
|
99
|
-
|
|
100
|
-
ok: arrived.hash === found.node.hash,
|
|
99
|
+
const walk = {
|
|
101
100
|
screen: found.name,
|
|
102
101
|
steps,
|
|
103
102
|
ranSteps: result.ranSteps,
|
|
104
103
|
results: result.results,
|
|
105
104
|
arrived: arrived.hash ? arrived.hash.slice(0, 8) : null,
|
|
106
105
|
};
|
|
106
|
+
if (arrived.hash === found.node.hash) return { ok: true, ...walk };
|
|
107
|
+
|
|
108
|
+
// A walk that ran and did not land had no name, and it was the only outcome
|
|
109
|
+
// here that did not.
|
|
110
|
+
//
|
|
111
|
+
// Every refusal above is named, logged and groupable; this one returned
|
|
112
|
+
// `{ok: false}` with no `reason` at all, so `simframe goto <known hash>` on
|
|
113
|
+
// CI failed the check that says *"it either walks there or names why it
|
|
114
|
+
// cannot"* with an empty detail — which is exactly what the check is for, and
|
|
115
|
+
// it had been sitting under an outcome nobody had named rather than under a
|
|
116
|
+
// crash.
|
|
117
|
+
//
|
|
118
|
+
// Two names, because they are two different faults and only one of them is
|
|
119
|
+
// about the graph. `route-halted` means a step on the route failed, which is
|
|
120
|
+
// an ordinary failure of the app or the moment. `arrived-elsewhere` means
|
|
121
|
+
// every step ran and we are *not where the graph promised* — an edge it
|
|
122
|
+
// remembers is wrong, and that is worth reading as a claim about memory
|
|
123
|
+
// rather than about this attempt.
|
|
124
|
+
const reason = !arrived.hash
|
|
125
|
+
? 'no-identity'
|
|
126
|
+
: (result.ranSteps < steps.length ? 'route-halted' : 'arrived-elsewhere');
|
|
127
|
+
return refuse(udid, { ok: false, reason, to: found.name, ...walk }, {
|
|
128
|
+
detail: reason === 'arrived-elsewhere'
|
|
129
|
+
? `route to "${found.name}" ran to the end and landed on ${walk.arrived}`
|
|
130
|
+
: `route to "${found.name}" stopped after ${result.ranSteps} of ${steps.length} step(s)`,
|
|
131
|
+
flowName: `goto:${target}`,
|
|
132
|
+
});
|
|
107
133
|
}
|
|
108
134
|
|
|
109
135
|
export function knownScreens(udid) {
|