simframe 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -1058
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +12 -6
- package/package.json +1 -1
- package/scripts/article-md.mjs +111 -45
- package/scripts/bench-hpi.mjs +53 -1
- package/scripts/ci-device-guard.mjs +27 -0
- package/scripts/ci-memory.mjs +33 -6
- package/scripts/demo-gif/README.md +36 -0
- package/scripts/demo-gif/compose.swift +106 -0
- package/scripts/demo-gif/events.example.json +74 -0
- package/scripts/demo-gif/flow.json +6 -0
- package/scripts/device-state.mjs +5 -62
- package/scripts/smithery/icon.png +0 -0
- package/scripts/smithery-bundle.mjs +77 -0
- package/src/actions.js +309 -32
- package/src/cli.js +111 -29
- package/src/device-state.js +101 -0
- package/src/index.js +139 -11
- package/src/metrics.js +144 -3
- package/src/navigate.js +10 -0
- package/src/platform/cdp.js +242 -0
- package/src/platform/index.js +28 -2
- package/src/store.js +39 -0
- package/src/wedge.js +370 -0
package/scripts/device-state.mjs
CHANGED
|
@@ -1,62 +1,5 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
|
|
6
|
-
// and the only thing that exercised it was a hosted runner, at the end of a
|
|
7
|
-
// fifteen-minute job, in the middle of a report. Two runtime bugs in this
|
|
8
|
-
// project came from logic that was correct and had never executed.
|
|
9
|
-
//
|
|
10
|
-
// Every entry here has been seen in a real run. Adding one from imagination is
|
|
11
|
-
// how a guard starts reviving genuine failures into passes.
|
|
12
|
-
|
|
13
|
-
/** Conditions that are the simulator, not the code. Each seen in a real run. */
|
|
14
|
-
export const DEVICE_STATE = [
|
|
15
|
-
[/NSPOSIXErrorDomain.*code=?\s*60|Operation timed out/i, 'simctl stopped answering (NSPOSIXErrorDomain 60)'],
|
|
16
|
-
[/did not produce a frame|produced no frame in \d+s/i, 'the daemon is up and the display renders nothing'],
|
|
17
|
-
[/Timeout waiting for screen surfaces|display surface is not answering|display surface could not be read/i, 'the display surface is wedged'],
|
|
18
|
-
[/no frames buffered|capture is wedged/i, 'capture stopped'],
|
|
19
|
-
[/the second app never launched|could not be dispatched/i, 'an app would not launch'],
|
|
20
|
-
// A launched app that never comes to the front, seen as the tour waiting for
|
|
21
|
-
// one of its landmarks on a screen that is showing a clock and nothing else.
|
|
22
|
-
//
|
|
23
|
-
// Measured on a runner: `ok launch — launched com.apple.Preferences
|
|
24
|
-
// (relaunched)` followed by `waited 8000ms for General: "General" is not on
|
|
25
|
-
// this screen. Visible: 10:50, .?o (the screen has not moved for 6181ms)`.
|
|
26
|
-
// Two labels, one of them a clock, on a still screen — the device is not
|
|
27
|
-
// presenting the app, and the guard called that a check failing on its
|
|
28
|
-
// merits and declined to revive.
|
|
29
|
-
//
|
|
30
|
-
// Deliberately narrow. It requires the wait to have failed AND the screen to
|
|
31
|
-
// have been still AND almost nothing readable: a tour that genuinely asks for
|
|
32
|
-
// the wrong label has a screen full of other labels, and must keep failing
|
|
33
|
-
// rather than being retried into a pass.
|
|
34
|
-
[
|
|
35
|
-
/never arrived[\s\S]*?Visible:[^\n]{0,24}\(the screen has not moved for \d+ms/i,
|
|
36
|
-
'a launched app never came to the front (the screen shows a clock and nothing else)',
|
|
37
|
-
],
|
|
38
|
-
// The same condition, now said outright by the step that suffered it instead
|
|
39
|
-
// of inferred from the shape of the screen afterwards. Item 169 gave `launch`
|
|
40
|
-
// a pid to compare, so a launch that starts a process and never fronts it
|
|
41
|
-
// reports itself; this signature fires on the cause rather than on a
|
|
42
|
-
// consequence that had to be recognised by "two labels, one a clock".
|
|
43
|
-
//
|
|
44
|
-
// It cannot be triggered by a tour asking for the wrong label — only a failed
|
|
45
|
-
// launch emits this sentence — so it needs none of the narrowing above.
|
|
46
|
-
[
|
|
47
|
-
/never came to the front within \d+ms/i,
|
|
48
|
-
'a launched app never came to the front (the launch said so itself, by pid)',
|
|
49
|
-
],
|
|
50
|
-
// Seen on the v0.14.3 bench run: `could not launch com.apple.Preferences:
|
|
51
|
-
// The system shell (SpringBoard:36454) probably crashed.` The guest's window
|
|
52
|
-
// server going down is the device, not the check, and nothing here matched it.
|
|
53
|
-
[
|
|
54
|
-
/system shell \(SpringBoard[^)]*\) probably crashed/i,
|
|
55
|
-
"the guest's SpringBoard crashed, so nothing can be fronted",
|
|
56
|
-
],
|
|
57
|
-
];
|
|
58
|
-
|
|
59
|
-
/** The condition this output shows, or null when the check failed on its merits. */
|
|
60
|
-
export function deviceCause(text) {
|
|
61
|
-
return DEVICE_STATE.find(([re]) => re.test(String(text ?? '')))?.[1] ?? null;
|
|
62
|
-
}
|
|
1
|
+
// Kept as a re-export: the table moved to `src/device-state.js` so `src/` can
|
|
2
|
+
// use it without importing out of `scripts/`. `ci-device-guard.mjs` and the
|
|
3
|
+
// unit test both reach it through this path, and a redirect is cheaper than
|
|
4
|
+
// updating every caller for a move that changes nothing about the table.
|
|
5
|
+
export { DEVICE_STATE, deviceCause } from '../src/device-state.js';
|
|
Binary file
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Builds the Smithery bundle for the current version:
|
|
3
|
+
// node scripts/smithery-bundle.mjs -> simframe-<version>.mcpb
|
|
4
|
+
// npx -y @smithery/cli mcp publish ./simframe-<version>.mcpb -n lvlr-xaus/simframe
|
|
5
|
+
//
|
|
6
|
+
// Smithery does not sync from the official MCP Registry, and its web form
|
|
7
|
+
// takes only an HTTPS URL, so a local stdio server reaches it as an MCPB
|
|
8
|
+
// bundle through the CLI. Two things learned publishing 0.18.0, both
|
|
9
|
+
// load-bearing:
|
|
10
|
+
//
|
|
11
|
+
// - Smithery's validator wants an `inputSchema` on every tool it is told
|
|
12
|
+
// about, and Anthropic's `mcpb pack` refuses that key as unknown. So the
|
|
13
|
+
// tool list is taken from the server's own tools/list answer, schemas
|
|
14
|
+
// included, and the archive is a plain zip — a .mcpb is nothing else.
|
|
15
|
+
// - The bundle is the packed npm tarball plus its runtime dependency
|
|
16
|
+
// installed inside it, because a bundle runs from its own directory.
|
|
17
|
+
import { execFileSync, spawn } from 'node:child_process';
|
|
18
|
+
import fs from 'node:fs';
|
|
19
|
+
import os from 'node:os';
|
|
20
|
+
import path from 'node:path';
|
|
21
|
+
import { fileURLToPath } from 'node:url';
|
|
22
|
+
|
|
23
|
+
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
24
|
+
const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
|
|
25
|
+
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'simframe-mcpb-'));
|
|
26
|
+
const dir = path.join(work, 'pkg');
|
|
27
|
+
fs.mkdirSync(dir);
|
|
28
|
+
|
|
29
|
+
const tarball = execFileSync('npm', ['pack', '--silent', '--pack-destination', work], { cwd: root }).toString().trim();
|
|
30
|
+
execFileSync('tar', ['-xzf', path.join(work, tarball), '-C', dir, '--strip-components=1']);
|
|
31
|
+
execFileSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', '--silent'], { cwd: dir, stdio: 'inherit' });
|
|
32
|
+
fs.copyFileSync(path.join(root, 'scripts/smithery/icon.png'), path.join(dir, 'icon.png'));
|
|
33
|
+
|
|
34
|
+
// The tool list, with schemas, from the server that ships in the bundle.
|
|
35
|
+
const tools = await new Promise((resolve, reject) => {
|
|
36
|
+
const child = spawn('node', [path.join(dir, 'src/cli.js'), 'mcp'], { cwd: dir, stdio: ['pipe', 'pipe', 'ignore'] });
|
|
37
|
+
let out = '';
|
|
38
|
+
child.stdout.on('data', (d) => { out += d; });
|
|
39
|
+
child.on('close', () => {
|
|
40
|
+
const line = out.split('\n').find((l) => l.includes('"id":2'));
|
|
41
|
+
line ? resolve(JSON.parse(line).result.tools) : reject(new Error('the bundled server did not answer tools/list'));
|
|
42
|
+
});
|
|
43
|
+
child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'bundle', version: '0' } } }) + '\n');
|
|
44
|
+
child.stdin.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');
|
|
45
|
+
child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: 2, method: 'tools/list' }) + '\n');
|
|
46
|
+
child.stdin.end();
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
const firstSentence = (s) => s.trim().split(/(?<=[.!?])\s/)[0].slice(0, 200);
|
|
50
|
+
const manifest = {
|
|
51
|
+
manifest_version: '0.3',
|
|
52
|
+
name: 'simframe',
|
|
53
|
+
display_name: 'simframe',
|
|
54
|
+
version: pkg.version,
|
|
55
|
+
description: 'Eyes, hands and memory for a coding agent driving the iOS Simulator or an Android emulator.',
|
|
56
|
+
long_description: 'simframe reads the simulator screen as a numbered text element map with tap points (accessibility tree + on-device OCR, ~20 ms warm frames instead of screenshots), runs whole tap/type/scroll/assert flows in one call with every step verified against what it did last time, and remembers screens so repeated flows need no model calls. Needs a Mac with Xcode; the first call builds a small Swift daemon from source (~15 s, once). Android emulators are driven with the same tools, without an accessibility tree.',
|
|
57
|
+
author: { name: 'Sadjad Asadi', url: 'https://github.com/lvlrSajjad' },
|
|
58
|
+
repository: { type: 'git', url: 'https://github.com/lvlrSajjad/simframe.git' },
|
|
59
|
+
homepage: 'https://lvlrsajjad.github.io/simframe/',
|
|
60
|
+
documentation: 'https://github.com/lvlrSajjad/simframe#readme',
|
|
61
|
+
support: 'https://github.com/lvlrSajjad/simframe/issues',
|
|
62
|
+
icon: 'icon.png',
|
|
63
|
+
server: { type: 'node', entry_point: 'src/cli.js', mcp_config: { command: 'node', args: ['${__dirname}/src/cli.js', 'mcp'] } },
|
|
64
|
+
tools: tools.map((t) => ({ name: t.name, description: firstSentence(t.description ?? ''), inputSchema: t.inputSchema ?? { type: 'object', properties: {} } })),
|
|
65
|
+
tools_generated: false,
|
|
66
|
+
keywords: pkg.keywords,
|
|
67
|
+
license: 'MIT',
|
|
68
|
+
compatibility: { platforms: ['darwin'], runtimes: { node: '>=18.17' } },
|
|
69
|
+
};
|
|
70
|
+
fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
71
|
+
|
|
72
|
+
const out = path.join(root, `simframe-${pkg.version}.mcpb`);
|
|
73
|
+
fs.rmSync(out, { force: true });
|
|
74
|
+
execFileSync('zip', ['-qr', out, '.', '-x', '*.DS_Store'], { cwd: dir });
|
|
75
|
+
fs.rmSync(work, { recursive: true, force: true });
|
|
76
|
+
console.log(`${path.relative(root, out)}: ${tools.length} tools, ${(fs.statSync(out).size / 1e6).toFixed(1)} MB`);
|
|
77
|
+
console.log(`publish: npx -y @smithery/cli mcp publish ./${path.relative(root, out)} -n lvlr-xaus/simframe`);
|
package/src/actions.js
CHANGED
|
@@ -16,6 +16,7 @@ import * as planner from './planner.js';
|
|
|
16
16
|
import * as metrics from './metrics.js';
|
|
17
17
|
import * as screenmap from './screenmap.js';
|
|
18
18
|
import { launchApp, openUrl, setPermission, terminateApp } from './platform/index.js';
|
|
19
|
+
import { shellCrashed } from './device-state.js';
|
|
19
20
|
|
|
20
21
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
21
22
|
const MAX_PAUSE_MS = 5000;
|
|
@@ -164,6 +165,59 @@ export function normalizeStep(raw) {
|
|
|
164
165
|
return step;
|
|
165
166
|
}
|
|
166
167
|
|
|
168
|
+
/**
|
|
169
|
+
* How long to keep waiting for a guest shell that has just crashed.
|
|
170
|
+
*
|
|
171
|
+
* Measured: six crashes on `326464A4`, every one of them recovered by waiting
|
|
172
|
+
* and launching again, in 5.7-7.9 s. 20 s is that with room, and it is bounded
|
|
173
|
+
* so a device that is genuinely gone still fails rather than hanging.
|
|
174
|
+
*/
|
|
175
|
+
export const SHELL_PATIENCE_MS = 20_000;
|
|
176
|
+
|
|
177
|
+
/** How often to try again while waiting for it. */
|
|
178
|
+
export const SHELL_POLL_MS = 1500;
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Launch, and survive the guest's window server dying under the attempt.
|
|
182
|
+
*
|
|
183
|
+
* `simctl` reports a SpringBoard crash as a launch failure, which is true and
|
|
184
|
+
* unhelpful: the shell is back a few seconds later and the same launch works.
|
|
185
|
+
* Until now the only remedy in the project was `simframe revive`, a ~40 s
|
|
186
|
+
* device restart — and item 173 notes that waiting and retrying "has never been
|
|
187
|
+
* tried". It was tried on 2026-09-18 and recovered 6 of 6.
|
|
188
|
+
*
|
|
189
|
+
* Deliberately narrow. Only the shell-crash signature is retried; every other
|
|
190
|
+
* device failure, including `simctl did not return within 90s`, fails on the
|
|
191
|
+
* first attempt as before, because none of them has been observed to heal and a
|
|
192
|
+
* retry on that one costs another 90 s to learn nothing.
|
|
193
|
+
*
|
|
194
|
+
* `report` is filled in rather than returned so the step can say it happened.
|
|
195
|
+
* A retry that does not show up in the summary is a rate nobody can argue with.
|
|
196
|
+
*/
|
|
197
|
+
export async function launchThroughShellCrash(
|
|
198
|
+
udid,
|
|
199
|
+
bundleId,
|
|
200
|
+
options,
|
|
201
|
+
report = {},
|
|
202
|
+
{ launch = launchApp, wait = (ms) => new Promise((r) => setTimeout(r, ms)), now = Date.now } = {},
|
|
203
|
+
) {
|
|
204
|
+
const started = now();
|
|
205
|
+
for (;;) {
|
|
206
|
+
report.attempts = (report.attempts ?? 0) + 1;
|
|
207
|
+
try {
|
|
208
|
+
const out = await launch(udid, bundleId, options);
|
|
209
|
+
report.waitedMs = now() - started;
|
|
210
|
+
return out;
|
|
211
|
+
} catch (err) {
|
|
212
|
+
if (!shellCrashed(err.message) || now() - started >= SHELL_PATIENCE_MS) {
|
|
213
|
+
report.waitedMs = now() - started;
|
|
214
|
+
throw err;
|
|
215
|
+
}
|
|
216
|
+
await wait(SHELL_POLL_MS);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
167
221
|
/**
|
|
168
222
|
* Does this verdict mean the flow went somewhere nobody intended?
|
|
169
223
|
*
|
|
@@ -740,6 +794,11 @@ export async function runScript(
|
|
|
740
794
|
verification = await confirmNoChange(deviceQuery, verification, {
|
|
741
795
|
beforeScreen, options, stableMs, timeoutMs,
|
|
742
796
|
});
|
|
797
|
+
// And the same courtesy before a wrong turn throws away the rest of
|
|
798
|
+
// the batch, which is the more expensive of the two mistakes.
|
|
799
|
+
verification = await confirmWrongTurn(deviceQuery, verification, {
|
|
800
|
+
udid, prediction, beforeScreen, step, options, stableMs, timeoutMs,
|
|
801
|
+
});
|
|
743
802
|
if (verification.lateArrival) {
|
|
744
803
|
// The reading the verdict was taken from is now known to be stale, so
|
|
745
804
|
// nothing downstream may learn a screen or an edge from it.
|
|
@@ -933,7 +992,19 @@ export async function runScript(
|
|
|
933
992
|
// switch moving 0.1% of the screen, which is neither faculty. Saying
|
|
934
993
|
// "assumed" is the honest answer; guessing a better-sounding reason
|
|
935
994
|
// would be the same mistake in the other direction.
|
|
936
|
-
|
|
995
|
+
//
|
|
996
|
+
// **Per verdict, though, not per site.** That argument is about
|
|
997
|
+
// `no-visible-change` and does not extend to every verdict: an
|
|
998
|
+
// `unexpected-screen` means the screen after the action was not the
|
|
999
|
+
// one memory predicted, which is item 174 and nothing else.
|
|
1000
|
+
// `metrics.VERDICT_FACULTY` holds the verdicts whose faculty is read,
|
|
1001
|
+
// and `no-visible-change` is deliberately not one of them — so this
|
|
1002
|
+
// still says "assumed" for the 162 records the reasoning above is
|
|
1003
|
+
// actually about, and stops saying it for the 26 it never covered.
|
|
1004
|
+
classified: Boolean(metrics.VERDICT_FACULTY[verification?.verdict]),
|
|
1005
|
+
// Recorded as a field rather than left as a prefix of `detail`, which
|
|
1006
|
+
// is how the breakdown had to recover it: by parsing a string.
|
|
1007
|
+
verdict: verification?.verdict ?? null,
|
|
937
1008
|
// `verification_failed` is the largest reason class in the log and it
|
|
938
1009
|
// was the only one carrying no intent, which made most of the corpus
|
|
939
1010
|
// useless for asking what kind of decision costs us. The step knows
|
|
@@ -989,6 +1060,11 @@ export async function runScript(
|
|
|
989
1060
|
const why = metrics.reasonForStepError(step, err);
|
|
990
1061
|
noteEscalation({
|
|
991
1062
|
stepIndex: i,
|
|
1063
|
+
// When the failure is the simulator rather than the code, say so. 78 of
|
|
1064
|
+
// the bench device's `verification_failed` records are `simctl` failing,
|
|
1065
|
+
// an app that would not launch, or capture stopping — item 173, counted
|
|
1066
|
+
// towards a perception phase.
|
|
1067
|
+
device: why.device ?? null,
|
|
992
1068
|
fingerprint: beforeScreen?.hash ?? metrics.fingerprintNow(udid, screenmap),
|
|
993
1069
|
reason: why.reason,
|
|
994
1070
|
candidates: why.candidates,
|
|
@@ -1758,6 +1834,77 @@ async function confirmNoChange(deviceQuery, verification, { beforeScreen, option
|
|
|
1758
1834
|
};
|
|
1759
1835
|
}
|
|
1760
1836
|
|
|
1837
|
+
/**
|
|
1838
|
+
* What a second look at a wrong turn establishes.
|
|
1839
|
+
*
|
|
1840
|
+
* Pure, and separate from the read that feeds it, because the read needs a
|
|
1841
|
+
* device and this needs to be provable without one. `stillOnPlan` is tested by
|
|
1842
|
+
* asserting against its own source text, which is what you write when the
|
|
1843
|
+
* decision cannot be reached in a test, and it checks that the code says what
|
|
1844
|
+
* it says rather than that it decides what it should.
|
|
1845
|
+
*
|
|
1846
|
+
* The rule: a second reading that *also* says wrong turn confirms the first,
|
|
1847
|
+
* and the halt stands on two reads instead of one. Any other second reading
|
|
1848
|
+
* replaces it, because the first was taken of a screen that had not finished
|
|
1849
|
+
* being itself. The disagreement is kept either way — it is the evidence that
|
|
1850
|
+
* this gate is flaky rather than protective, and it is what a supervisor would
|
|
1851
|
+
* be asked to rule on.
|
|
1852
|
+
*/
|
|
1853
|
+
export function afterSecondLook(verification, second, again) {
|
|
1854
|
+
if (!second?.verdict || second.verdict === 'unexpected-screen') return verification;
|
|
1855
|
+
const was = String(verification?.observed?.to ?? '').slice(0, 8);
|
|
1856
|
+
const now = String(again?.hash ?? '').slice(0, 8);
|
|
1857
|
+
return {
|
|
1858
|
+
...verification,
|
|
1859
|
+
verdict: second.verdict,
|
|
1860
|
+
detail: `${second.detail} — read again after settling, because the first read said`
|
|
1861
|
+
+ ` "${verification.detail}"${was && now && was !== now ? ` (${was} → ${now})` : ''}`,
|
|
1862
|
+
// Both answers, so the log can count how often perception contradicts
|
|
1863
|
+
// itself here rather than only how often it halted a run.
|
|
1864
|
+
disagreed: { first: verification.verdict, second: second.verdict, from: was || null, to: now || null },
|
|
1865
|
+
lateArrival: again,
|
|
1866
|
+
};
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
/**
|
|
1870
|
+
* One more look before a wrong turn is allowed to stop the batch.
|
|
1871
|
+
*
|
|
1872
|
+
* The sibling of `confirmNoChange`, for the same reason and against the same
|
|
1873
|
+
* class of mistake: a verdict taken from a screen that was still arriving. The
|
|
1874
|
+
* reported symptom is that the gate is **non-deterministic** — a reporter
|
|
1875
|
+
* re-issued the identical call with no state change and it passed — and a gate
|
|
1876
|
+
* that fails once and passes on retry is flaky, not protective. Asked twice,
|
|
1877
|
+
* the perception disagrees with itself, and the first answer was never
|
|
1878
|
+
* evidence.
|
|
1879
|
+
*
|
|
1880
|
+
* This keeps the verify barrier rather than softening it. The barrier asks for
|
|
1881
|
+
* confirmed perception before acting on a screen an `unexpected-*` verdict has
|
|
1882
|
+
* touched; a halt on one premature read is not confirmed perception either.
|
|
1883
|
+
* Both directions now cost a second, settled read, and a wrong turn that is
|
|
1884
|
+
* real still halts — on better evidence than before.
|
|
1885
|
+
*
|
|
1886
|
+
* Deliberately not gated on a supervisor being configured. The escape hatch the
|
|
1887
|
+
* item proposed was a supervisor ruling, which `doctor` reports as
|
|
1888
|
+
* `none — not requested` on a default install, so that fix would have reached
|
|
1889
|
+
* only callers who opted in. A second read needs no model and no opt-in; a
|
|
1890
|
+
* supervisor, when there is one, still gets asked about what survives it.
|
|
1891
|
+
*/
|
|
1892
|
+
async function confirmWrongTurn(deviceQuery, verification, { udid, prediction, beforeScreen, step, options, stableMs, timeoutMs }) {
|
|
1893
|
+
if (verification?.verdict !== 'unexpected-screen' || !prediction || !beforeScreen?.hash) return verification;
|
|
1894
|
+
await api.waitFor(deviceQuery, {
|
|
1895
|
+
mode: 'settle', stableMs: 250, timeoutMs: LATE_CHANGE_MS, options,
|
|
1896
|
+
}).catch(() => null);
|
|
1897
|
+
const again = await api.screenIdentity(deviceQuery, { options, settleMs: stableMs, timeoutMs }).catch(() => null);
|
|
1898
|
+
// Nothing looked, so nothing was established and the first answer stands.
|
|
1899
|
+
// A failed read may never be read as agreement.
|
|
1900
|
+
if (!again?.hash) return verification;
|
|
1901
|
+
// `kind` is deliberately not passed: the transition classifier's reading
|
|
1902
|
+
// belongs to the moment of the first look, and it only ever decorates an
|
|
1903
|
+
// `ok`. Re-using it here would date-stamp this answer with that one.
|
|
1904
|
+
const second = graph.verdict({ udid, prediction, before: beforeScreen, after: again, action: step.action });
|
|
1905
|
+
return afterSecondLook(verification, second, again);
|
|
1906
|
+
}
|
|
1907
|
+
|
|
1761
1908
|
/**
|
|
1762
1909
|
* A control that changed state is not a step that did nothing.
|
|
1763
1910
|
*
|
|
@@ -2699,11 +2846,12 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2699
2846
|
return `pressed key ${step.value ?? step.code}`;
|
|
2700
2847
|
case 'launch': {
|
|
2701
2848
|
const bundleId = step.value ?? step.bundleId;
|
|
2702
|
-
const
|
|
2849
|
+
const shell = { waitedMs: 0, attempts: 0 };
|
|
2850
|
+
const started = await launchThroughShellCrash(udid, bundleId, {
|
|
2703
2851
|
args: step.args ?? [],
|
|
2704
2852
|
env: step.env ?? {},
|
|
2705
2853
|
terminateFirst: step.relaunch === true,
|
|
2706
|
-
});
|
|
2854
|
+
}, shell);
|
|
2707
2855
|
// Item 169: simctl returning ok means the process started, not that the
|
|
2708
2856
|
// app came forward, and the two came apart repeatedly on a loaded
|
|
2709
2857
|
// runner — leaving the device on the previous app under a step that
|
|
@@ -2743,6 +2891,10 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2743
2891
|
if (ctx.landing) ctx.landing.verdict = landed.verdict;
|
|
2744
2892
|
return `launched ${bundleId}${step.relaunch ? ' (relaunched)' : ''}`
|
|
2745
2893
|
+ (landed.verdict === 'fronted' ? ` (frontmost after ${landed.ms}ms)` : '')
|
|
2894
|
+
+ (shell.attempts > 1
|
|
2895
|
+
? ` [the guest's SpringBoard crashed and came back; launched again after ${shell.waitedMs}ms`
|
|
2896
|
+
+ `, on attempt ${shell.attempts}]`
|
|
2897
|
+
: '')
|
|
2746
2898
|
+ (retried
|
|
2747
2899
|
? ` [it did not front on the first attempt — ${frontmost.describeHeld(retried.held, started.pid)}`
|
|
2748
2900
|
+ `; a second launch fronted it. The launch is unreliable on this host.]`
|
|
@@ -2856,6 +3008,30 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2856
3008
|
// `direction` still wins, for a caller who knows better.
|
|
2857
3009
|
const asked = step.direction ? String(step.direction).toLowerCase() : null;
|
|
2858
3010
|
const max = Math.min(MAX_SCROLLS, step.maxScrolls ?? 6);
|
|
3011
|
+
// **The screen's size, in scope.** It was not, and that single omission is
|
|
3012
|
+
// the whole of item 190 and of the field report's Defect 2.
|
|
3013
|
+
//
|
|
3014
|
+
// `points` existed only inside `offsetSays`'s own destructure, so every
|
|
3015
|
+
// other reference to it — `inViewport`, and the message that reports a
|
|
3016
|
+
// target as out of view — was an undeclared identifier. `inViewport` runs
|
|
3017
|
+
// on **every successful locate**, so the moment this step found what it
|
|
3018
|
+
// was looking for it threw `ReferenceError: points is not defined`, the
|
|
3019
|
+
// loop's own `catch` swallowed it as "not found", and it scrolled on until
|
|
3020
|
+
// the budget ran out and it reported the target unreachable.
|
|
3021
|
+
//
|
|
3022
|
+
// So `scrollTo` could fail on a target that was on screen, could fail
|
|
3023
|
+
// after a scroll that had just brought the target into view, and could
|
|
3024
|
+
// burn its whole budget doing it — all three of the reported symptoms,
|
|
3025
|
+
// and all three are this. Measured from the loop, i is the iteration:
|
|
3026
|
+
//
|
|
3027
|
+
// i=0 locate threw: "Developer" is not on this screen (true, it was above)
|
|
3028
|
+
// i=1 locate threw: points is not defined (it had FOUND it)
|
|
3029
|
+
// i=2 locate threw: points is not defined (again)
|
|
3030
|
+
//
|
|
3031
|
+
// Optional chaining does not help: `points?.width` on an undeclared name
|
|
3032
|
+
// is still a ReferenceError, which is why nothing caught this by reading.
|
|
3033
|
+
const geo = await ctx.screen();
|
|
3034
|
+
const points = { width: geo.pointWidth, height: geo.pointHeight };
|
|
2859
3035
|
let dir = asked ?? 'down';
|
|
2860
3036
|
let reversed = false;
|
|
2861
3037
|
// Where the target is, when the tree knows. `null` means no evidence, and
|
|
@@ -2863,18 +3039,40 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2863
3039
|
// the top of a web page, which **triggers pull-to-refresh**, reloads the
|
|
2864
3040
|
// page and changes the screen hash — defeating the end-detection below
|
|
2865
3041
|
// and reading, from outside, as an endless loop. Observed live.
|
|
2866
|
-
|
|
2867
|
-
|
|
3042
|
+
//
|
|
3043
|
+
// **And it answers `null` far more often than this was written for.**
|
|
3044
|
+
// Measured on 2026-09-18, which is item 190: iOS puts only the *visible*
|
|
3045
|
+
// rows of a table in the accessibility tree. A scrolled-away row is not
|
|
3046
|
+
// there with an out-of-bounds coordinate — it is absent.
|
|
3047
|
+
//
|
|
3048
|
+
// Settings at the top, resolve("Developer") -> status=none, y=undefined
|
|
3049
|
+
// Settings at the bottom, resolve("Accessibility") -> status=none, y=undefined
|
|
3050
|
+
//
|
|
3051
|
+
// Zero elements whose label even contains the query, both times. So on a
|
|
3052
|
+
// native list this returns `null` in precisely the case it exists to
|
|
3053
|
+
// answer, and the y<0 / y>height reasoning below only ever fires on the
|
|
3054
|
+
// frameworks that do keep off-screen rows in the tree — React Native, and
|
|
3055
|
+
// the `y = -693` case this was written for.
|
|
3056
|
+
// One read answers both questions, so asking them costs what asking one
|
|
3057
|
+
// used to: which way is the target, and what is on screen right now.
|
|
3058
|
+
const lookAround = async () => {
|
|
2868
3059
|
try {
|
|
2869
|
-
const { entry
|
|
2870
|
-
const
|
|
3060
|
+
const { entry } = await api.readScreenWith(deviceQuery, { useOcr: false, options: ctx.options });
|
|
3061
|
+
const targets = entry.targets ?? [];
|
|
3062
|
+
// What is visible, as one comparable string. This is the end detector
|
|
3063
|
+
// now, and the frame hash is not — see `scrolledNowhere`.
|
|
3064
|
+
const signature = targets.map((t) => String(t.label ?? '')).filter(Boolean).sort().join('\u0000');
|
|
3065
|
+
if (asked) return { says: asked, signature };
|
|
3066
|
+
const hit = matching.resolve(targets, String(query));
|
|
2871
3067
|
const y = hit?.target?.y;
|
|
2872
|
-
|
|
2873
|
-
if (y
|
|
2874
|
-
if (y
|
|
2875
|
-
|
|
3068
|
+
let says = null;
|
|
3069
|
+
if (!Number.isFinite(y)) says = null;
|
|
3070
|
+
else if (y < 0) says = 'up';
|
|
3071
|
+
else if (y > (points?.height ?? Infinity)) says = 'down';
|
|
3072
|
+
else says = dir;
|
|
3073
|
+
return { says, signature };
|
|
2876
3074
|
} catch {
|
|
2877
|
-
return null;
|
|
3075
|
+
return { says: asked ?? null, signature: null };
|
|
2878
3076
|
}
|
|
2879
3077
|
};
|
|
2880
3078
|
/**
|
|
@@ -2907,25 +3105,57 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2907
3105
|
// rather than the act — reported as "after 1 scroll down" on a request
|
|
2908
3106
|
// for "up".
|
|
2909
3107
|
const scrolled = [];
|
|
3108
|
+
const arrivedHow = () => (scrolled.length
|
|
3109
|
+
? ` after ${scrolled.length} scroll${scrolled.length === 1 ? '' : 's'} `
|
|
3110
|
+
+ (new Set(scrolled).size === 1 ? scrolled[0] : scrolled.join(' then '))
|
|
3111
|
+
: ' already');
|
|
3112
|
+
/**
|
|
3113
|
+
* Is it here now? Asked without throwing, so a give-up path can use it.
|
|
3114
|
+
*
|
|
3115
|
+
* **A frame hash is a whole-screen measure and a strip is a small part of
|
|
3116
|
+
* the screen**, so "the frame did not change" is not "nothing moved".
|
|
3117
|
+
* Reported from the field: a horizontally-scrolling tab strip had the
|
|
3118
|
+
* target scrolled into view — the visible tabs demonstrably changed — and
|
|
3119
|
+
* this step reported `not reachable by scrolling` anyway, because the
|
|
3120
|
+
* whole-frame hash barely moved. A confident wrong *failure* after the
|
|
3121
|
+
* action succeeded, which costs what a silent success costs: the caller
|
|
3122
|
+
* is handed a false fact and a round trip.
|
|
3123
|
+
*
|
|
3124
|
+
* Same principle as the recall path (DEFERRED 184): a miss may not be the
|
|
3125
|
+
* final word until something has looked.
|
|
3126
|
+
*/
|
|
3127
|
+
const nowInView = async () => {
|
|
3128
|
+
try {
|
|
3129
|
+
const found = await api.locate(deviceQuery, query, { index: step.index, refresh: true });
|
|
3130
|
+
return inViewport(found.target) ? found : null;
|
|
3131
|
+
} catch {
|
|
3132
|
+
return null;
|
|
3133
|
+
}
|
|
3134
|
+
};
|
|
3135
|
+
// What the last look actually established, so the give-up message can say
|
|
3136
|
+
// what it knows instead of guessing. `absent` means the resolver did not
|
|
3137
|
+
// find it; `off-view` means it did and the target sits outside the
|
|
3138
|
+
// viewport — and in that case claiming it may not be in the tree is
|
|
3139
|
+
// flatly contradicted by the tree we just read.
|
|
3140
|
+
let lastMiss = null;
|
|
2910
3141
|
for (let i = 0; i <= max; i += 1) {
|
|
2911
3142
|
try {
|
|
2912
3143
|
const found = await api.locate(deviceQuery, query, { index: step.index, refresh: i > 0 });
|
|
2913
|
-
if (!inViewport(found.target))
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${how}`;
|
|
3144
|
+
if (!inViewport(found.target)) {
|
|
3145
|
+
lastMiss = 'off-view';
|
|
3146
|
+
throw new Error(
|
|
3147
|
+
`"${query}" is in the tree but not in view (at ${found.target.x},${found.target.y}`
|
|
3148
|
+
+ ` on a ${Math.round(points?.width ?? 0)}x${Math.round(points?.height ?? 0)}pt screen)`);
|
|
3149
|
+
}
|
|
3150
|
+
return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${arrivedHow()}`;
|
|
2921
3151
|
} catch (err) {
|
|
3152
|
+
if (lastMiss !== 'off-view') lastMiss = 'absent';
|
|
2922
3153
|
if (i === max) {
|
|
2923
3154
|
throw new Error(`scrolled ${dir} ${max}x without finding ${query}: ${err.message}`);
|
|
2924
3155
|
}
|
|
2925
3156
|
}
|
|
2926
|
-
const evidence = await
|
|
3157
|
+
const { says: evidence, signature: wasShowing } = await lookAround();
|
|
2927
3158
|
if (evidence) dir = evidence;
|
|
2928
|
-
const wasAt = await hashNow(deviceQuery, ctx.options);
|
|
2929
3159
|
scrolled.push(dir);
|
|
2930
3160
|
await runStep(deviceQuery, udid, { action: 'scroll', value: dir }, ctx);
|
|
2931
3161
|
// A scroll either moves immediately or not at all, so it does not need a
|
|
@@ -2939,17 +3169,64 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2939
3169
|
// page"*. Reverse once — the target may be behind us, and on a page
|
|
2940
3170
|
// whose fields never enter the tree there is no offset to follow — then
|
|
2941
3171
|
// stop rather than thrash.
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
3172
|
+
// **What "it stopped moving" is measured on, and the frame hash was the
|
|
3173
|
+
// wrong thing.** At the end of a list iOS rubber-bands: the content does
|
|
3174
|
+
// not advance and the pixels do, so a whole-frame comparison answers
|
|
3175
|
+
// "yes, it moved" on every attempt and the end is never detected. Seen
|
|
3176
|
+
// on 2026-09-18 — `scrolled down 6x without finding Accessibility` on a
|
|
3177
|
+
// list that had been against its bottom stop the whole time, 21.5s for
|
|
3178
|
+
// nothing.
|
|
3179
|
+
//
|
|
3180
|
+
// The visible labels do not bounce. If the same rows are on screen after
|
|
3181
|
+
// a scroll as before it, the scroll achieved nothing, whatever the
|
|
3182
|
+
// framebuffer did.
|
|
3183
|
+
const { signature: nowShowing } = await lookAround();
|
|
3184
|
+
if (wasShowing && nowShowing && wasShowing === nowShowing) {
|
|
3185
|
+
// Before believing the hash, look. See `nowInView` — an unchanged
|
|
3186
|
+
// whole-frame hash is weak evidence about a strip, and this step has
|
|
3187
|
+
// reported failure on a scroll that worked.
|
|
3188
|
+
const arrived = await nowInView();
|
|
3189
|
+
if (arrived) {
|
|
3190
|
+
return `"${arrived.target.label ?? query}" is in view at ${arrived.target.x},${arrived.target.y}`
|
|
3191
|
+
+ `${arrivedHow()} (the frame hash did not register the scroll — looked again rather than`
|
|
3192
|
+
+ ' reporting a failure the screen contradicts)';
|
|
3193
|
+
}
|
|
3194
|
+
// **A stall is evidence, and treating it as none is item 190.**
|
|
3195
|
+
//
|
|
3196
|
+
// This used to reverse only when the *tree* said where the target
|
|
3197
|
+
// was — and on a native list the tree never does, because scrolled-
|
|
3198
|
+
// away rows are not in it. So the common case was: no evidence, the
|
|
3199
|
+
// default direction, and if that direction happened to be the wrong
|
|
3200
|
+
// one the step burned its budget moving away from the target and then
|
|
3201
|
+
// refused, having never looked the other way. Field-measured at 12.6s
|
|
3202
|
+
// and 18.4s for zero progress, the largest single latency cost in
|
|
3203
|
+
// that round.
|
|
3204
|
+
//
|
|
3205
|
+
// Having scrolled and moved nothing, we know this direction is
|
|
3206
|
+
// exhausted. That is a fact about the screen rather than a guess, and
|
|
3207
|
+
// it is a different claim from the one the rule above was protecting
|
|
3208
|
+
// against: the danger there was *opening* with an unfounded "up",
|
|
3209
|
+
// which walks a web page to the top and pulls it to refresh. Here we
|
|
3210
|
+
// are at one end and the only place left to look is the other. Still
|
|
3211
|
+
// exactly once, still bounded by `max`.
|
|
3212
|
+
if (reversed) {
|
|
2948
3213
|
throw new Error(
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
3214
|
+
// Both ways, always — this branch is now only reached after a
|
|
3215
|
+
// reversal, so "there is no direction to try" (which it used to
|
|
3216
|
+
// say when the tree was silent) is no longer true and would be
|
|
3217
|
+
// the same kind of unchecked assertion as the sentence below it.
|
|
3218
|
+
`${query} is not reachable by scrolling: it stopped moving both ways`
|
|
3219
|
+
+ ` (${scrolled.join(', ')}) after ${i + 1} attempt(s).`
|
|
3220
|
+
// Say what was established, not what would be convenient. This
|
|
3221
|
+
// used to assert "It may not be in the accessibility tree at all"
|
|
3222
|
+
// in every case — a claim the function never checks, and one the
|
|
3223
|
+
// `off-view` branch has already disproved by reading the element
|
|
3224
|
+
// out of the tree a line earlier.
|
|
3225
|
+
+ (lastMiss === 'off-view'
|
|
3226
|
+
? ' It IS in the tree and outside the viewport, so this is a scrolling problem'
|
|
3227
|
+
+ ' rather than a perception one — try a different container, or aim at a coordinate.'
|
|
3228
|
+
: ' The resolver did not find it here either, so it may not be on this screen at all —'
|
|
3229
|
+
+ ' read the screen (sim_ui), or aim at a coordinate.'),
|
|
2953
3230
|
);
|
|
2954
3231
|
}
|
|
2955
3232
|
reversed = true;
|