simframe 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,6 @@
1
+ [
2
+ { "tap": "Accessibility" },
3
+ { "tap": "Display & Text Size" },
4
+ { "tap": "Larger Text" },
5
+ { "assert": { "value": "Larger Text" } }
6
+ ]
@@ -1,62 +1,5 @@
1
- // Is a failed CI step the device's fault or the check's?
2
- //
3
- // Its own module, and the reason is the same one that moved `classifyStray` out
4
- // of `eval-fingerprint.mjs`: `ci-device-guard.mjs` reads `process.argv` and
5
- // `process.exit(2)`s at import, so nothing could ever test this table there —
6
- // and the only thing that exercised it was a hosted runner, at the end of a
7
- // fifteen-minute job, in the middle of a report. Two runtime bugs in this
8
- // project came from logic that was correct and had never executed.
9
- //
10
- // Every entry here has been seen in a real run. Adding one from imagination is
11
- // how a guard starts reviving genuine failures into passes.
12
-
13
- /** Conditions that are the simulator, not the code. Each seen in a real run. */
14
- export const DEVICE_STATE = [
15
- [/NSPOSIXErrorDomain.*code=?\s*60|Operation timed out/i, 'simctl stopped answering (NSPOSIXErrorDomain 60)'],
16
- [/did not produce a frame|produced no frame in \d+s/i, 'the daemon is up and the display renders nothing'],
17
- [/Timeout waiting for screen surfaces|display surface is not answering|display surface could not be read/i, 'the display surface is wedged'],
18
- [/no frames buffered|capture is wedged/i, 'capture stopped'],
19
- [/the second app never launched|could not be dispatched/i, 'an app would not launch'],
20
- // A launched app that never comes to the front, seen as the tour waiting for
21
- // one of its landmarks on a screen that is showing a clock and nothing else.
22
- //
23
- // Measured on a runner: `ok launch — launched com.apple.Preferences
24
- // (relaunched)` followed by `waited 8000ms for General: "General" is not on
25
- // this screen. Visible: 10:50, .?o (the screen has not moved for 6181ms)`.
26
- // Two labels, one of them a clock, on a still screen — the device is not
27
- // presenting the app, and the guard called that a check failing on its
28
- // merits and declined to revive.
29
- //
30
- // Deliberately narrow. It requires the wait to have failed AND the screen to
31
- // have been still AND almost nothing readable: a tour that genuinely asks for
32
- // the wrong label has a screen full of other labels, and must keep failing
33
- // rather than being retried into a pass.
34
- [
35
- /never arrived[\s\S]*?Visible:[^\n]{0,24}\(the screen has not moved for \d+ms/i,
36
- 'a launched app never came to the front (the screen shows a clock and nothing else)',
37
- ],
38
- // The same condition, now said outright by the step that suffered it instead
39
- // of inferred from the shape of the screen afterwards. Item 169 gave `launch`
40
- // a pid to compare, so a launch that starts a process and never fronts it
41
- // reports itself; this signature fires on the cause rather than on a
42
- // consequence that had to be recognised by "two labels, one a clock".
43
- //
44
- // It cannot be triggered by a tour asking for the wrong label — only a failed
45
- // launch emits this sentence — so it needs none of the narrowing above.
46
- [
47
- /never came to the front within \d+ms/i,
48
- 'a launched app never came to the front (the launch said so itself, by pid)',
49
- ],
50
- // Seen on the v0.14.3 bench run: `could not launch com.apple.Preferences:
51
- // The system shell (SpringBoard:36454) probably crashed.` The guest's window
52
- // server going down is the device, not the check, and nothing here matched it.
53
- [
54
- /system shell \(SpringBoard[^)]*\) probably crashed/i,
55
- "the guest's SpringBoard crashed, so nothing can be fronted",
56
- ],
57
- ];
58
-
59
- /** The condition this output shows, or null when the check failed on its merits. */
60
- export function deviceCause(text) {
61
- return DEVICE_STATE.find(([re]) => re.test(String(text ?? '')))?.[1] ?? null;
62
- }
1
+ // Kept as a re-export: the table moved to `src/device-state.js` so `src/` can
2
+ // use it without importing out of `scripts/`. `ci-device-guard.mjs` and the
3
+ // unit test both reach it through this path, and a redirect is cheaper than
4
+ // updating every caller for a move that changes nothing about the table.
5
+ export { DEVICE_STATE, deviceCause } from '../src/device-state.js';
Binary file
@@ -0,0 +1,77 @@
1
+ #!/usr/bin/env node
2
+ // Builds the Smithery bundle for the current version:
3
+ // node scripts/smithery-bundle.mjs -> simframe-<version>.mcpb
4
+ // npx -y @smithery/cli mcp publish ./simframe-<version>.mcpb -n lvlr-xaus/simframe
5
+ //
6
+ // Smithery does not sync from the official MCP Registry, and its web form
7
+ // takes only an HTTPS URL, so a local stdio server reaches it as an MCPB
8
+ // bundle through the CLI. Two things learned publishing 0.18.0, both
9
+ // load-bearing:
10
+ //
11
+ // - Smithery's validator wants an `inputSchema` on every tool it is told
12
+ // about, and Anthropic's `mcpb pack` refuses that key as unknown. So the
13
+ // tool list is taken from the server's own tools/list answer, schemas
14
+ // included, and the archive is a plain zip — a .mcpb is nothing else.
15
+ // - The bundle is the packed npm tarball plus its runtime dependency
16
+ // installed inside it, because a bundle runs from its own directory.
17
+ import { execFileSync, spawn } from 'node:child_process';
18
+ import fs from 'node:fs';
19
+ import os from 'node:os';
20
+ import path from 'node:path';
21
+ import { fileURLToPath } from 'node:url';
22
+
23
+ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
24
+ const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
25
+ const work = fs.mkdtempSync(path.join(os.tmpdir(), 'simframe-mcpb-'));
26
+ const dir = path.join(work, 'pkg');
27
+ fs.mkdirSync(dir);
28
+
29
+ const tarball = execFileSync('npm', ['pack', '--silent', '--pack-destination', work], { cwd: root }).toString().trim();
30
+ execFileSync('tar', ['-xzf', path.join(work, tarball), '-C', dir, '--strip-components=1']);
31
+ execFileSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', '--silent'], { cwd: dir, stdio: 'inherit' });
32
+ fs.copyFileSync(path.join(root, 'scripts/smithery/icon.png'), path.join(dir, 'icon.png'));
33
+
34
+ // The tool list, with schemas, from the server that ships in the bundle.
35
+ const tools = await new Promise((resolve, reject) => {
36
+ const child = spawn('node', [path.join(dir, 'src/cli.js'), 'mcp'], { cwd: dir, stdio: ['pipe', 'pipe', 'ignore'] });
37
+ let out = '';
38
+ child.stdout.on('data', (d) => { out += d; });
39
+ child.on('close', () => {
40
+ const line = out.split('\n').find((l) => l.includes('"id":2'));
41
+ line ? resolve(JSON.parse(line).result.tools) : reject(new Error('the bundled server did not answer tools/list'));
42
+ });
43
+ child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'bundle', version: '0' } } }) + '\n');
44
+ child.stdin.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');
45
+ child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: 2, method: 'tools/list' }) + '\n');
46
+ child.stdin.end();
47
+ });
48
+
49
+ const firstSentence = (s) => s.trim().split(/(?<=[.!?])\s/)[0].slice(0, 200);
50
+ const manifest = {
51
+ manifest_version: '0.3',
52
+ name: 'simframe',
53
+ display_name: 'simframe',
54
+ version: pkg.version,
55
+ description: 'Eyes, hands and memory for a coding agent driving the iOS Simulator or an Android emulator.',
56
+ long_description: 'simframe reads the simulator screen as a numbered text element map with tap points (accessibility tree + on-device OCR, ~20 ms warm frames instead of screenshots), runs whole tap/type/scroll/assert flows in one call with every step verified against what it did last time, and remembers screens so repeated flows need no model calls. Needs a Mac with Xcode; the first call builds a small Swift daemon from source (~15 s, once). Android emulators are driven with the same tools, without an accessibility tree.',
57
+ author: { name: 'Sadjad Asadi', url: 'https://github.com/lvlrSajjad' },
58
+ repository: { type: 'git', url: 'https://github.com/lvlrSajjad/simframe.git' },
59
+ homepage: 'https://lvlrsajjad.github.io/simframe/',
60
+ documentation: 'https://github.com/lvlrSajjad/simframe#readme',
61
+ support: 'https://github.com/lvlrSajjad/simframe/issues',
62
+ icon: 'icon.png',
63
+ server: { type: 'node', entry_point: 'src/cli.js', mcp_config: { command: 'node', args: ['${__dirname}/src/cli.js', 'mcp'] } },
64
+ tools: tools.map((t) => ({ name: t.name, description: firstSentence(t.description ?? ''), inputSchema: t.inputSchema ?? { type: 'object', properties: {} } })),
65
+ tools_generated: false,
66
+ keywords: pkg.keywords,
67
+ license: 'MIT',
68
+ compatibility: { platforms: ['darwin'], runtimes: { node: '>=18.17' } },
69
+ };
70
+ fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest, null, 2));
71
+
72
+ const out = path.join(root, `simframe-${pkg.version}.mcpb`);
73
+ fs.rmSync(out, { force: true });
74
+ execFileSync('zip', ['-qr', out, '.', '-x', '*.DS_Store'], { cwd: dir });
75
+ fs.rmSync(work, { recursive: true, force: true });
76
+ console.log(`${path.relative(root, out)}: ${tools.length} tools, ${(fs.statSync(out).size / 1e6).toFixed(1)} MB`);
77
+ console.log(`publish: npx -y @smithery/cli mcp publish ./${path.relative(root, out)} -n lvlr-xaus/simframe`);
package/src/actions.js CHANGED
@@ -16,6 +16,7 @@ import * as planner from './planner.js';
16
16
  import * as metrics from './metrics.js';
17
17
  import * as screenmap from './screenmap.js';
18
18
  import { launchApp, openUrl, setPermission, terminateApp } from './platform/index.js';
19
+ import { shellCrashed } from './device-state.js';
19
20
 
20
21
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
21
22
  const MAX_PAUSE_MS = 5000;
@@ -164,6 +165,59 @@ export function normalizeStep(raw) {
164
165
  return step;
165
166
  }
166
167
 
168
+ /**
169
+ * How long to keep waiting for a guest shell that has just crashed.
170
+ *
171
+ * Measured: six crashes on `326464A4`, every one of them recovered by waiting
172
+ * and launching again, in 5.7-7.9 s. 20 s is that with room, and it is bounded
173
+ * so a device that is genuinely gone still fails rather than hanging.
174
+ */
175
+ export const SHELL_PATIENCE_MS = 20_000;
176
+
177
+ /** How often to try again while waiting for it. */
178
+ export const SHELL_POLL_MS = 1500;
179
+
180
+ /**
181
+ * Launch, and survive the guest's window server dying under the attempt.
182
+ *
183
+ * `simctl` reports a SpringBoard crash as a launch failure, which is true and
184
+ * unhelpful: the shell is back a few seconds later and the same launch works.
185
+ * Until now the only remedy in the project was `simframe revive`, a ~40 s
186
+ * device restart — and item 173 notes that waiting and retrying "has never been
187
+ * tried". It was tried on 2026-09-18 and recovered 6 of 6.
188
+ *
189
+ * Deliberately narrow. Only the shell-crash signature is retried; every other
190
+ * device failure, including `simctl did not return within 90s`, fails on the
191
+ * first attempt as before, because none of them has been observed to heal and a
192
+ * retry on that one costs another 90 s to learn nothing.
193
+ *
194
+ * `report` is filled in rather than returned so the step can say it happened.
195
+ * A retry that does not show up in the summary is a rate nobody can argue with.
196
+ */
197
+ export async function launchThroughShellCrash(
198
+ udid,
199
+ bundleId,
200
+ options,
201
+ report = {},
202
+ { launch = launchApp, wait = (ms) => new Promise((r) => setTimeout(r, ms)), now = Date.now } = {},
203
+ ) {
204
+ const started = now();
205
+ for (;;) {
206
+ report.attempts = (report.attempts ?? 0) + 1;
207
+ try {
208
+ const out = await launch(udid, bundleId, options);
209
+ report.waitedMs = now() - started;
210
+ return out;
211
+ } catch (err) {
212
+ if (!shellCrashed(err.message) || now() - started >= SHELL_PATIENCE_MS) {
213
+ report.waitedMs = now() - started;
214
+ throw err;
215
+ }
216
+ await wait(SHELL_POLL_MS);
217
+ }
218
+ }
219
+ }
220
+
167
221
  /**
168
222
  * Does this verdict mean the flow went somewhere nobody intended?
169
223
  *
@@ -740,6 +794,11 @@ export async function runScript(
740
794
  verification = await confirmNoChange(deviceQuery, verification, {
741
795
  beforeScreen, options, stableMs, timeoutMs,
742
796
  });
797
+ // And the same courtesy before a wrong turn throws away the rest of
798
+ // the batch, which is the more expensive of the two mistakes.
799
+ verification = await confirmWrongTurn(deviceQuery, verification, {
800
+ udid, prediction, beforeScreen, step, options, stableMs, timeoutMs,
801
+ });
743
802
  if (verification.lateArrival) {
744
803
  // The reading the verdict was taken from is now known to be stale, so
745
804
  // nothing downstream may learn a screen or an edge from it.
@@ -933,7 +992,19 @@ export async function runScript(
933
992
  // switch moving 0.1% of the screen, which is neither faculty. Saying
934
993
  // "assumed" is the honest answer; guessing a better-sounding reason
935
994
  // would be the same mistake in the other direction.
936
- classified: false,
995
+ //
996
+ // **Per verdict, though, not per site.** That argument is about
997
+ // `no-visible-change` and does not extend to every verdict: an
998
+ // `unexpected-screen` means the screen after the action was not the
999
+ // one memory predicted, which is item 174 and nothing else.
1000
+ // `metrics.VERDICT_FACULTY` holds the verdicts whose faculty is read,
1001
+ // and `no-visible-change` is deliberately not one of them — so this
1002
+ // still says "assumed" for the 162 records the reasoning above is
1003
+ // actually about, and stops saying it for the 26 it never covered.
1004
+ classified: Boolean(metrics.VERDICT_FACULTY[verification?.verdict]),
1005
+ // Recorded as a field rather than left as a prefix of `detail`, which
1006
+ // is how the breakdown had to recover it: by parsing a string.
1007
+ verdict: verification?.verdict ?? null,
937
1008
  // `verification_failed` is the largest reason class in the log and it
938
1009
  // was the only one carrying no intent, which made most of the corpus
939
1010
  // useless for asking what kind of decision costs us. The step knows
@@ -989,6 +1060,11 @@ export async function runScript(
989
1060
  const why = metrics.reasonForStepError(step, err);
990
1061
  noteEscalation({
991
1062
  stepIndex: i,
1063
+ // When the failure is the simulator rather than the code, say so. 78 of
1064
+ // the bench device's `verification_failed` records are `simctl` failing,
1065
+ // an app that would not launch, or capture stopping — item 173, counted
1066
+ // towards a perception phase.
1067
+ device: why.device ?? null,
992
1068
  fingerprint: beforeScreen?.hash ?? metrics.fingerprintNow(udid, screenmap),
993
1069
  reason: why.reason,
994
1070
  candidates: why.candidates,
@@ -1758,6 +1834,77 @@ async function confirmNoChange(deviceQuery, verification, { beforeScreen, option
1758
1834
  };
1759
1835
  }
1760
1836
 
1837
+ /**
1838
+ * What a second look at a wrong turn establishes.
1839
+ *
1840
+ * Pure, and separate from the read that feeds it, because the read needs a
1841
+ * device and this needs to be provable without one. `stillOnPlan` is tested by
1842
+ * asserting against its own source text, which is what you write when the
1843
+ * decision cannot be reached in a test, and it checks that the code says what
1844
+ * it says rather than that it decides what it should.
1845
+ *
1846
+ * The rule: a second reading that *also* says wrong turn confirms the first,
1847
+ * and the halt stands on two reads instead of one. Any other second reading
1848
+ * replaces it, because the first was taken of a screen that had not finished
1849
+ * being itself. The disagreement is kept either way — it is the evidence that
1850
+ * this gate is flaky rather than protective, and it is what a supervisor would
1851
+ * be asked to rule on.
1852
+ */
1853
+ export function afterSecondLook(verification, second, again) {
1854
+ if (!second?.verdict || second.verdict === 'unexpected-screen') return verification;
1855
+ const was = String(verification?.observed?.to ?? '').slice(0, 8);
1856
+ const now = String(again?.hash ?? '').slice(0, 8);
1857
+ return {
1858
+ ...verification,
1859
+ verdict: second.verdict,
1860
+ detail: `${second.detail} — read again after settling, because the first read said`
1861
+ + ` "${verification.detail}"${was && now && was !== now ? ` (${was} → ${now})` : ''}`,
1862
+ // Both answers, so the log can count how often perception contradicts
1863
+ // itself here rather than only how often it halted a run.
1864
+ disagreed: { first: verification.verdict, second: second.verdict, from: was || null, to: now || null },
1865
+ lateArrival: again,
1866
+ };
1867
+ }
1868
+
1869
+ /**
1870
+ * One more look before a wrong turn is allowed to stop the batch.
1871
+ *
1872
+ * The sibling of `confirmNoChange`, for the same reason and against the same
1873
+ * class of mistake: a verdict taken from a screen that was still arriving. The
1874
+ * reported symptom is that the gate is **non-deterministic** — a reporter
1875
+ * re-issued the identical call with no state change and it passed — and a gate
1876
+ * that fails once and passes on retry is flaky, not protective. Asked twice,
1877
+ * the perception disagrees with itself, and the first answer was never
1878
+ * evidence.
1879
+ *
1880
+ * This keeps the verify barrier rather than softening it. The barrier asks for
1881
+ * confirmed perception before acting on a screen an `unexpected-*` verdict has
1882
+ * touched; a halt on one premature read is not confirmed perception either.
1883
+ * Both directions now cost a second, settled read, and a wrong turn that is
1884
+ * real still halts — on better evidence than before.
1885
+ *
1886
+ * Deliberately not gated on a supervisor being configured. The escape hatch the
1887
+ * item proposed was a supervisor ruling, which `doctor` reports as
1888
+ * `none — not requested` on a default install, so that fix would have reached
1889
+ * only callers who opted in. A second read needs no model and no opt-in; a
1890
+ * supervisor, when there is one, still gets asked about what survives it.
1891
+ */
1892
+ async function confirmWrongTurn(deviceQuery, verification, { udid, prediction, beforeScreen, step, options, stableMs, timeoutMs }) {
1893
+ if (verification?.verdict !== 'unexpected-screen' || !prediction || !beforeScreen?.hash) return verification;
1894
+ await api.waitFor(deviceQuery, {
1895
+ mode: 'settle', stableMs: 250, timeoutMs: LATE_CHANGE_MS, options,
1896
+ }).catch(() => null);
1897
+ const again = await api.screenIdentity(deviceQuery, { options, settleMs: stableMs, timeoutMs }).catch(() => null);
1898
+ // Nothing looked, so nothing was established and the first answer stands.
1899
+ // A failed read may never be read as agreement.
1900
+ if (!again?.hash) return verification;
1901
+ // `kind` is deliberately not passed: the transition classifier's reading
1902
+ // belongs to the moment of the first look, and it only ever decorates an
1903
+ // `ok`. Re-using it here would date-stamp this answer with that one.
1904
+ const second = graph.verdict({ udid, prediction, before: beforeScreen, after: again, action: step.action });
1905
+ return afterSecondLook(verification, second, again);
1906
+ }
1907
+
1761
1908
  /**
1762
1909
  * A control that changed state is not a step that did nothing.
1763
1910
  *
@@ -2699,11 +2846,12 @@ async function runStep(deviceQuery, udid, step, ctx) {
2699
2846
  return `pressed key ${step.value ?? step.code}`;
2700
2847
  case 'launch': {
2701
2848
  const bundleId = step.value ?? step.bundleId;
2702
- const started = await launchApp(udid, bundleId, {
2849
+ const shell = { waitedMs: 0, attempts: 0 };
2850
+ const started = await launchThroughShellCrash(udid, bundleId, {
2703
2851
  args: step.args ?? [],
2704
2852
  env: step.env ?? {},
2705
2853
  terminateFirst: step.relaunch === true,
2706
- });
2854
+ }, shell);
2707
2855
  // Item 169: simctl returning ok means the process started, not that the
2708
2856
  // app came forward, and the two came apart repeatedly on a loaded
2709
2857
  // runner — leaving the device on the previous app under a step that
@@ -2743,6 +2891,10 @@ async function runStep(deviceQuery, udid, step, ctx) {
2743
2891
  if (ctx.landing) ctx.landing.verdict = landed.verdict;
2744
2892
  return `launched ${bundleId}${step.relaunch ? ' (relaunched)' : ''}`
2745
2893
  + (landed.verdict === 'fronted' ? ` (frontmost after ${landed.ms}ms)` : '')
2894
+ + (shell.attempts > 1
2895
+ ? ` [the guest's SpringBoard crashed and came back; launched again after ${shell.waitedMs}ms`
2896
+ + `, on attempt ${shell.attempts}]`
2897
+ : '')
2746
2898
  + (retried
2747
2899
  ? ` [it did not front on the first attempt — ${frontmost.describeHeld(retried.held, started.pid)}`
2748
2900
  + `; a second launch fronted it. The launch is unreliable on this host.]`
@@ -2856,6 +3008,30 @@ async function runStep(deviceQuery, udid, step, ctx) {
2856
3008
  // `direction` still wins, for a caller who knows better.
2857
3009
  const asked = step.direction ? String(step.direction).toLowerCase() : null;
2858
3010
  const max = Math.min(MAX_SCROLLS, step.maxScrolls ?? 6);
3011
+ // **The screen's size, in scope.** It was not, and that single omission is
3012
+ // the whole of item 190 and of the field report's Defect 2.
3013
+ //
3014
+ // `points` existed only inside `offsetSays`'s own destructure, so every
3015
+ // other reference to it — `inViewport`, and the message that reports a
3016
+ // target as out of view — was an undeclared identifier. `inViewport` runs
3017
+ // on **every successful locate**, so the moment this step found what it
3018
+ // was looking for it threw `ReferenceError: points is not defined`, the
3019
+ // loop's own `catch` swallowed it as "not found", and it scrolled on until
3020
+ // the budget ran out and it reported the target unreachable.
3021
+ //
3022
+ // So `scrollTo` could fail on a target that was on screen, could fail
3023
+ // after a scroll that had just brought the target into view, and could
3024
+ // burn its whole budget doing it — all three of the reported symptoms,
3025
+ // and all three are this. Measured from the loop, i is the iteration:
3026
+ //
3027
+ // i=0 locate threw: "Developer" is not on this screen (true, it was above)
3028
+ // i=1 locate threw: points is not defined (it had FOUND it)
3029
+ // i=2 locate threw: points is not defined (again)
3030
+ //
3031
+ // Optional chaining does not help: `points?.width` on an undeclared name
3032
+ // is still a ReferenceError, which is why nothing caught this by reading.
3033
+ const geo = await ctx.screen();
3034
+ const points = { width: geo.pointWidth, height: geo.pointHeight };
2859
3035
  let dir = asked ?? 'down';
2860
3036
  let reversed = false;
2861
3037
  // Where the target is, when the tree knows. `null` means no evidence, and
@@ -2863,18 +3039,40 @@ async function runStep(deviceQuery, udid, step, ctx) {
2863
3039
  // the top of a web page, which **triggers pull-to-refresh**, reloads the
2864
3040
  // page and changes the screen hash — defeating the end-detection below
2865
3041
  // and reading, from outside, as an endless loop. Observed live.
2866
- const offsetSays = async () => {
2867
- if (asked) return asked;
3042
+ //
3043
+ // **And it answers `null` far more often than this was written for.**
3044
+ // Measured on 2026-09-18, which is item 190: iOS puts only the *visible*
3045
+ // rows of a table in the accessibility tree. A scrolled-away row is not
3046
+ // there with an out-of-bounds coordinate — it is absent.
3047
+ //
3048
+ // Settings at the top, resolve("Developer") -> status=none, y=undefined
3049
+ // Settings at the bottom, resolve("Accessibility") -> status=none, y=undefined
3050
+ //
3051
+ // Zero elements whose label even contains the query, both times. So on a
3052
+ // native list this returns `null` in precisely the case it exists to
3053
+ // answer, and the y<0 / y>height reasoning below only ever fires on the
3054
+ // frameworks that do keep off-screen rows in the tree — React Native, and
3055
+ // the `y = -693` case this was written for.
3056
+ // One read answers both questions, so asking them costs what asking one
3057
+ // used to: which way is the target, and what is on screen right now.
3058
+ const lookAround = async () => {
2868
3059
  try {
2869
- const { entry, points } = await api.readScreenWith(deviceQuery, { useOcr: false, options: ctx.options });
2870
- const hit = matching.resolve(entry.targets ?? [], String(query));
3060
+ const { entry } = await api.readScreenWith(deviceQuery, { useOcr: false, options: ctx.options });
3061
+ const targets = entry.targets ?? [];
3062
+ // What is visible, as one comparable string. This is the end detector
3063
+ // now, and the frame hash is not — see `scrolledNowhere`.
3064
+ const signature = targets.map((t) => String(t.label ?? '')).filter(Boolean).sort().join('\u0000');
3065
+ if (asked) return { says: asked, signature };
3066
+ const hit = matching.resolve(targets, String(query));
2871
3067
  const y = hit?.target?.y;
2872
- if (!Number.isFinite(y)) return null;
2873
- if (y < 0) return 'up';
2874
- if (y > (points?.height ?? Infinity)) return 'down';
2875
- return dir;
3068
+ let says = null;
3069
+ if (!Number.isFinite(y)) says = null;
3070
+ else if (y < 0) says = 'up';
3071
+ else if (y > (points?.height ?? Infinity)) says = 'down';
3072
+ else says = dir;
3073
+ return { says, signature };
2876
3074
  } catch {
2877
- return null;
3075
+ return { says: asked ?? null, signature: null };
2878
3076
  }
2879
3077
  };
2880
3078
  /**
@@ -2907,25 +3105,57 @@ async function runStep(deviceQuery, udid, step, ctx) {
2907
3105
  // rather than the act — reported as "after 1 scroll down" on a request
2908
3106
  // for "up".
2909
3107
  const scrolled = [];
3108
+ const arrivedHow = () => (scrolled.length
3109
+ ? ` after ${scrolled.length} scroll${scrolled.length === 1 ? '' : 's'} `
3110
+ + (new Set(scrolled).size === 1 ? scrolled[0] : scrolled.join(' then '))
3111
+ : ' already');
3112
+ /**
3113
+ * Is it here now? Asked without throwing, so a give-up path can use it.
3114
+ *
3115
+ * **A frame hash is a whole-screen measure and a strip is a small part of
3116
+ * the screen**, so "the frame did not change" is not "nothing moved".
3117
+ * Reported from the field: a horizontally-scrolling tab strip had the
3118
+ * target scrolled into view — the visible tabs demonstrably changed — and
3119
+ * this step reported `not reachable by scrolling` anyway, because the
3120
+ * whole-frame hash barely moved. A confident wrong *failure* after the
3121
+ * action succeeded, which costs what a silent success costs: the caller
3122
+ * is handed a false fact and a round trip.
3123
+ *
3124
+ * Same principle as the recall path (DEFERRED 184): a miss may not be the
3125
+ * final word until something has looked.
3126
+ */
3127
+ const nowInView = async () => {
3128
+ try {
3129
+ const found = await api.locate(deviceQuery, query, { index: step.index, refresh: true });
3130
+ return inViewport(found.target) ? found : null;
3131
+ } catch {
3132
+ return null;
3133
+ }
3134
+ };
3135
+ // What the last look actually established, so the give-up message can say
3136
+ // what it knows instead of guessing. `absent` means the resolver did not
3137
+ // find it; `off-view` means it did and the target sits outside the
3138
+ // viewport — and in that case claiming it may not be in the tree is
3139
+ // flatly contradicted by the tree we just read.
3140
+ let lastMiss = null;
2910
3141
  for (let i = 0; i <= max; i += 1) {
2911
3142
  try {
2912
3143
  const found = await api.locate(deviceQuery, query, { index: step.index, refresh: i > 0 });
2913
- if (!inViewport(found.target)) throw new Error(
2914
- `"${query}" is in the tree but not in view (at ${found.target.x},${found.target.y}`
2915
- + ` on a ${Math.round(points?.width ?? 0)}x${Math.round(points?.height ?? 0)}pt screen)`);
2916
- const how = scrolled.length
2917
- ? ` after ${scrolled.length} scroll${scrolled.length === 1 ? '' : 's'} `
2918
- + (new Set(scrolled).size === 1 ? scrolled[0] : scrolled.join(' then '))
2919
- : ' already';
2920
- return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${how}`;
3144
+ if (!inViewport(found.target)) {
3145
+ lastMiss = 'off-view';
3146
+ throw new Error(
3147
+ `"${query}" is in the tree but not in view (at ${found.target.x},${found.target.y}`
3148
+ + ` on a ${Math.round(points?.width ?? 0)}x${Math.round(points?.height ?? 0)}pt screen)`);
3149
+ }
3150
+ return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${arrivedHow()}`;
2921
3151
  } catch (err) {
3152
+ if (lastMiss !== 'off-view') lastMiss = 'absent';
2922
3153
  if (i === max) {
2923
3154
  throw new Error(`scrolled ${dir} ${max}x without finding ${query}: ${err.message}`);
2924
3155
  }
2925
3156
  }
2926
- const evidence = await offsetSays();
3157
+ const { says: evidence, signature: wasShowing } = await lookAround();
2927
3158
  if (evidence) dir = evidence;
2928
- const wasAt = await hashNow(deviceQuery, ctx.options);
2929
3159
  scrolled.push(dir);
2930
3160
  await runStep(deviceQuery, udid, { action: 'scroll', value: dir }, ctx);
2931
3161
  // A scroll either moves immediately or not at all, so it does not need a
@@ -2939,17 +3169,64 @@ async function runStep(deviceQuery, udid, step, ctx) {
2939
3169
  // page"*. Reverse once — the target may be behind us, and on a page
2940
3170
  // whose fields never enter the tree there is no offset to follow — then
2941
3171
  // stop rather than thrash.
2942
- const nowAt = await hashNow(deviceQuery, ctx.options);
2943
- if (wasAt && nowAt && wasAt === nowAt) {
2944
- // Reverse only on evidence. Without it we do not know the target is
2945
- // behind us, and scrolling blindly the other way is how a web page
2946
- // gets pulled to refresh.
2947
- if (reversed || !evidence) {
3172
+ // **What "it stopped moving" is measured on, and the frame hash was the
3173
+ // wrong thing.** At the end of a list iOS rubber-bands: the content does
3174
+ // not advance and the pixels do, so a whole-frame comparison answers
3175
+ // "yes, it moved" on every attempt and the end is never detected. Seen
3176
+ // on 2026-09-18 — `scrolled down 6x without finding Accessibility` on a
3177
+ // list that had been against its bottom stop the whole time, 21.5s for
3178
+ // nothing.
3179
+ //
3180
+ // The visible labels do not bounce. If the same rows are on screen after
3181
+ // a scroll as before it, the scroll achieved nothing, whatever the
3182
+ // framebuffer did.
3183
+ const { signature: nowShowing } = await lookAround();
3184
+ if (wasShowing && nowShowing && wasShowing === nowShowing) {
3185
+ // Before believing the hash, look. See `nowInView` — an unchanged
3186
+ // whole-frame hash is weak evidence about a strip, and this step has
3187
+ // reported failure on a scroll that worked.
3188
+ const arrived = await nowInView();
3189
+ if (arrived) {
3190
+ return `"${arrived.target.label ?? query}" is in view at ${arrived.target.x},${arrived.target.y}`
3191
+ + `${arrivedHow()} (the frame hash did not register the scroll — looked again rather than`
3192
+ + ' reporting a failure the screen contradicts)';
3193
+ }
3194
+ // **A stall is evidence, and treating it as none is item 190.**
3195
+ //
3196
+ // This used to reverse only when the *tree* said where the target
3197
+ // was — and on a native list the tree never does, because scrolled-
3198
+ // away rows are not in it. So the common case was: no evidence, the
3199
+ // default direction, and if that direction happened to be the wrong
3200
+ // one the step burned its budget moving away from the target and then
3201
+ // refused, having never looked the other way. Field-measured at 12.6s
3202
+ // and 18.4s for zero progress, the largest single latency cost in
3203
+ // that round.
3204
+ //
3205
+ // Having scrolled and moved nothing, we know this direction is
3206
+ // exhausted. That is a fact about the screen rather than a guess, and
3207
+ // it is a different claim from the one the rule above was protecting
3208
+ // against: the danger there was *opening* with an unfounded "up",
3209
+ // which walks a web page to the top and pulls it to refresh. Here we
3210
+ // are at one end and the only place left to look is the other. Still
3211
+ // exactly once, still bounded by `max`.
3212
+ if (reversed) {
2948
3213
  throw new Error(
2949
- `${query} is not reachable by scrolling: ${dir} stopped moving after ${i + 1} attempt(s)`
2950
- + (evidence ? ' and so did the other way.' : ' and the tree does not say where it is,'
2951
- + ' so there is no direction to try.')
2952
- + ' It may not be in the accessibility tree at all — read the screen, or aim at a coordinate.',
3214
+ // Both ways, always — this branch is now only reached after a
3215
+ // reversal, so "there is no direction to try" (which it used to
3216
+ // say when the tree was silent) is no longer true and would be
3217
+ // the same kind of unchecked assertion as the sentence below it.
3218
+ `${query} is not reachable by scrolling: it stopped moving both ways`
3219
+ + ` (${scrolled.join(', ')}) after ${i + 1} attempt(s).`
3220
+ // Say what was established, not what would be convenient. This
3221
+ // used to assert "It may not be in the accessibility tree at all"
3222
+ // in every case — a claim the function never checks, and one the
3223
+ // `off-view` branch has already disproved by reading the element
3224
+ // out of the tree a line earlier.
3225
+ + (lastMiss === 'off-view'
3226
+ ? ' It IS in the tree and outside the viewport, so this is a scrolling problem'
3227
+ + ' rather than a perception one — try a different container, or aim at a coordinate.'
3228
+ : ' The resolver did not find it here either, so it may not be on this screen at all —'
3229
+ + ' read the screen (sim_ui), or aim at a coordinate.'),
2953
3230
  );
2954
3231
  }
2955
3232
  reversed = true;