simframe 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -1106
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +12 -6
- package/package.json +1 -1
- package/scripts/article-md.mjs +111 -45
- package/scripts/bench-hpi.mjs +45 -1
- package/scripts/ci-memory.mjs +33 -6
- package/scripts/demo-gif/README.md +36 -0
- package/scripts/demo-gif/compose.swift +106 -0
- package/scripts/demo-gif/events.example.json +74 -0
- package/scripts/demo-gif/flow.json +6 -0
- package/scripts/smithery/icon.png +0 -0
- package/scripts/smithery-bundle.mjs +77 -0
- package/src/actions.js +291 -31
- package/src/cli.js +45 -26
- package/src/device-state.js +37 -0
- package/src/index.js +139 -11
- package/src/platform/cdp.js +242 -0
- package/src/platform/index.js +28 -2
- package/src/store.js +39 -0
- package/src/wedge.js +154 -2
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Builds the Smithery bundle for the current version:
|
|
3
|
+
// node scripts/smithery-bundle.mjs -> simframe-<version>.mcpb
|
|
4
|
+
// npx -y @smithery/cli mcp publish ./simframe-<version>.mcpb -n lvlr-xaus/simframe
|
|
5
|
+
//
|
|
6
|
+
// Smithery does not sync from the official MCP Registry, and its web form
|
|
7
|
+
// takes only an HTTPS URL, so a local stdio server reaches it as an MCPB
|
|
8
|
+
// bundle through the CLI. Two things learned publishing 0.18.0, both
|
|
9
|
+
// load-bearing:
|
|
10
|
+
//
|
|
11
|
+
// - Smithery's validator wants an `inputSchema` on every tool it is told
|
|
12
|
+
// about, and Anthropic's `mcpb pack` refuses that key as unknown. So the
|
|
13
|
+
// tool list is taken from the server's own tools/list answer, schemas
|
|
14
|
+
// included, and the archive is a plain zip — a .mcpb is nothing else.
|
|
15
|
+
// - The bundle is the packed npm tarball plus its runtime dependency
|
|
16
|
+
// installed inside it, because a bundle runs from its own directory.
|
|
17
|
+
import { execFileSync, spawn } from 'node:child_process';
|
|
18
|
+
import fs from 'node:fs';
|
|
19
|
+
import os from 'node:os';
|
|
20
|
+
import path from 'node:path';
|
|
21
|
+
import { fileURLToPath } from 'node:url';
|
|
22
|
+
|
|
23
|
+
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
24
|
+
const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
|
|
25
|
+
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'simframe-mcpb-'));
|
|
26
|
+
const dir = path.join(work, 'pkg');
|
|
27
|
+
fs.mkdirSync(dir);
|
|
28
|
+
|
|
29
|
+
const tarball = execFileSync('npm', ['pack', '--silent', '--pack-destination', work], { cwd: root }).toString().trim();
|
|
30
|
+
execFileSync('tar', ['-xzf', path.join(work, tarball), '-C', dir, '--strip-components=1']);
|
|
31
|
+
execFileSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', '--silent'], { cwd: dir, stdio: 'inherit' });
|
|
32
|
+
fs.copyFileSync(path.join(root, 'scripts/smithery/icon.png'), path.join(dir, 'icon.png'));
|
|
33
|
+
|
|
34
|
+
// The tool list, with schemas, from the server that ships in the bundle.
|
|
35
|
+
const tools = await new Promise((resolve, reject) => {
|
|
36
|
+
const child = spawn('node', [path.join(dir, 'src/cli.js'), 'mcp'], { cwd: dir, stdio: ['pipe', 'pipe', 'ignore'] });
|
|
37
|
+
let out = '';
|
|
38
|
+
child.stdout.on('data', (d) => { out += d; });
|
|
39
|
+
child.on('close', () => {
|
|
40
|
+
const line = out.split('\n').find((l) => l.includes('"id":2'));
|
|
41
|
+
line ? resolve(JSON.parse(line).result.tools) : reject(new Error('the bundled server did not answer tools/list'));
|
|
42
|
+
});
|
|
43
|
+
child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'bundle', version: '0' } } }) + '\n');
|
|
44
|
+
child.stdin.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');
|
|
45
|
+
child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: 2, method: 'tools/list' }) + '\n');
|
|
46
|
+
child.stdin.end();
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
const firstSentence = (s) => s.trim().split(/(?<=[.!?])\s/)[0].slice(0, 200);
|
|
50
|
+
const manifest = {
|
|
51
|
+
manifest_version: '0.3',
|
|
52
|
+
name: 'simframe',
|
|
53
|
+
display_name: 'simframe',
|
|
54
|
+
version: pkg.version,
|
|
55
|
+
description: 'Eyes, hands and memory for a coding agent driving the iOS Simulator or an Android emulator.',
|
|
56
|
+
long_description: 'simframe reads the simulator screen as a numbered text element map with tap points (accessibility tree + on-device OCR, ~20 ms warm frames instead of screenshots), runs whole tap/type/scroll/assert flows in one call with every step verified against what it did last time, and remembers screens so repeated flows need no model calls. Needs a Mac with Xcode; the first call builds a small Swift daemon from source (~15 s, once). Android emulators are driven with the same tools, without an accessibility tree.',
|
|
57
|
+
author: { name: 'Sadjad Asadi', url: 'https://github.com/lvlrSajjad' },
|
|
58
|
+
repository: { type: 'git', url: 'https://github.com/lvlrSajjad/simframe.git' },
|
|
59
|
+
homepage: 'https://lvlrsajjad.github.io/simframe/',
|
|
60
|
+
documentation: 'https://github.com/lvlrSajjad/simframe#readme',
|
|
61
|
+
support: 'https://github.com/lvlrSajjad/simframe/issues',
|
|
62
|
+
icon: 'icon.png',
|
|
63
|
+
server: { type: 'node', entry_point: 'src/cli.js', mcp_config: { command: 'node', args: ['${__dirname}/src/cli.js', 'mcp'] } },
|
|
64
|
+
tools: tools.map((t) => ({ name: t.name, description: firstSentence(t.description ?? ''), inputSchema: t.inputSchema ?? { type: 'object', properties: {} } })),
|
|
65
|
+
tools_generated: false,
|
|
66
|
+
keywords: pkg.keywords,
|
|
67
|
+
license: 'MIT',
|
|
68
|
+
compatibility: { platforms: ['darwin'], runtimes: { node: '>=18.17' } },
|
|
69
|
+
};
|
|
70
|
+
fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
71
|
+
|
|
72
|
+
const out = path.join(root, `simframe-${pkg.version}.mcpb`);
|
|
73
|
+
fs.rmSync(out, { force: true });
|
|
74
|
+
execFileSync('zip', ['-qr', out, '.', '-x', '*.DS_Store'], { cwd: dir });
|
|
75
|
+
fs.rmSync(work, { recursive: true, force: true });
|
|
76
|
+
console.log(`${path.relative(root, out)}: ${tools.length} tools, ${(fs.statSync(out).size / 1e6).toFixed(1)} MB`);
|
|
77
|
+
console.log(`publish: npx -y @smithery/cli mcp publish ./${path.relative(root, out)} -n lvlr-xaus/simframe`);
|
package/src/actions.js
CHANGED
|
@@ -16,6 +16,7 @@ import * as planner from './planner.js';
|
|
|
16
16
|
import * as metrics from './metrics.js';
|
|
17
17
|
import * as screenmap from './screenmap.js';
|
|
18
18
|
import { launchApp, openUrl, setPermission, terminateApp } from './platform/index.js';
|
|
19
|
+
import { shellCrashed } from './device-state.js';
|
|
19
20
|
|
|
20
21
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
21
22
|
const MAX_PAUSE_MS = 5000;
|
|
@@ -164,6 +165,59 @@ export function normalizeStep(raw) {
|
|
|
164
165
|
return step;
|
|
165
166
|
}
|
|
166
167
|
|
|
168
|
+
/**
|
|
169
|
+
* How long to keep waiting for a guest shell that has just crashed.
|
|
170
|
+
*
|
|
171
|
+
* Measured: six crashes on `326464A4`, every one of them recovered by waiting
|
|
172
|
+
* and launching again, in 5.7-7.9 s. 20 s is that with room, and it is bounded
|
|
173
|
+
* so a device that is genuinely gone still fails rather than hanging.
|
|
174
|
+
*/
|
|
175
|
+
export const SHELL_PATIENCE_MS = 20_000;
|
|
176
|
+
|
|
177
|
+
/** How often to try again while waiting for it. */
|
|
178
|
+
export const SHELL_POLL_MS = 1500;
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Launch, and survive the guest's window server dying under the attempt.
|
|
182
|
+
*
|
|
183
|
+
* `simctl` reports a SpringBoard crash as a launch failure, which is true and
|
|
184
|
+
* unhelpful: the shell is back a few seconds later and the same launch works.
|
|
185
|
+
* Until now the only remedy in the project was `simframe revive`, a ~40 s
|
|
186
|
+
* device restart — and item 173 notes that waiting and retrying "has never been
|
|
187
|
+
* tried". It was tried on 2026-09-18 and recovered 6 of 6.
|
|
188
|
+
*
|
|
189
|
+
* Deliberately narrow. Only the shell-crash signature is retried; every other
|
|
190
|
+
* device failure, including `simctl did not return within 90s`, fails on the
|
|
191
|
+
* first attempt as before, because none of them has been observed to heal and a
|
|
192
|
+
* retry on that one costs another 90 s to learn nothing.
|
|
193
|
+
*
|
|
194
|
+
* `report` is filled in rather than returned so the step can say it happened.
|
|
195
|
+
* A retry that does not show up in the summary is a rate nobody can argue with.
|
|
196
|
+
*/
|
|
197
|
+
export async function launchThroughShellCrash(
|
|
198
|
+
udid,
|
|
199
|
+
bundleId,
|
|
200
|
+
options,
|
|
201
|
+
report = {},
|
|
202
|
+
{ launch = launchApp, wait = (ms) => new Promise((r) => setTimeout(r, ms)), now = Date.now } = {},
|
|
203
|
+
) {
|
|
204
|
+
const started = now();
|
|
205
|
+
for (;;) {
|
|
206
|
+
report.attempts = (report.attempts ?? 0) + 1;
|
|
207
|
+
try {
|
|
208
|
+
const out = await launch(udid, bundleId, options);
|
|
209
|
+
report.waitedMs = now() - started;
|
|
210
|
+
return out;
|
|
211
|
+
} catch (err) {
|
|
212
|
+
if (!shellCrashed(err.message) || now() - started >= SHELL_PATIENCE_MS) {
|
|
213
|
+
report.waitedMs = now() - started;
|
|
214
|
+
throw err;
|
|
215
|
+
}
|
|
216
|
+
await wait(SHELL_POLL_MS);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
167
221
|
/**
|
|
168
222
|
* Does this verdict mean the flow went somewhere nobody intended?
|
|
169
223
|
*
|
|
@@ -740,6 +794,11 @@ export async function runScript(
|
|
|
740
794
|
verification = await confirmNoChange(deviceQuery, verification, {
|
|
741
795
|
beforeScreen, options, stableMs, timeoutMs,
|
|
742
796
|
});
|
|
797
|
+
// And the same courtesy before a wrong turn throws away the rest of
|
|
798
|
+
// the batch, which is the more expensive of the two mistakes.
|
|
799
|
+
verification = await confirmWrongTurn(deviceQuery, verification, {
|
|
800
|
+
udid, prediction, beforeScreen, step, options, stableMs, timeoutMs,
|
|
801
|
+
});
|
|
743
802
|
if (verification.lateArrival) {
|
|
744
803
|
// The reading the verdict was taken from is now known to be stale, so
|
|
745
804
|
// nothing downstream may learn a screen or an edge from it.
|
|
@@ -1775,6 +1834,77 @@ async function confirmNoChange(deviceQuery, verification, { beforeScreen, option
|
|
|
1775
1834
|
};
|
|
1776
1835
|
}
|
|
1777
1836
|
|
|
1837
|
+
/**
|
|
1838
|
+
* What a second look at a wrong turn establishes.
|
|
1839
|
+
*
|
|
1840
|
+
* Pure, and separate from the read that feeds it, because the read needs a
|
|
1841
|
+
* device and this needs to be provable without one. `stillOnPlan` is tested by
|
|
1842
|
+
* asserting against its own source text, which is what you write when the
|
|
1843
|
+
* decision cannot be reached in a test, and it checks that the code says what
|
|
1844
|
+
* it says rather than that it decides what it should.
|
|
1845
|
+
*
|
|
1846
|
+
* The rule: a second reading that *also* says wrong turn confirms the first,
|
|
1847
|
+
* and the halt stands on two reads instead of one. Any other second reading
|
|
1848
|
+
* replaces it, because the first was taken of a screen that had not finished
|
|
1849
|
+
* being itself. The disagreement is kept either way — it is the evidence that
|
|
1850
|
+
* this gate is flaky rather than protective, and it is what a supervisor would
|
|
1851
|
+
* be asked to rule on.
|
|
1852
|
+
*/
|
|
1853
|
+
export function afterSecondLook(verification, second, again) {
|
|
1854
|
+
if (!second?.verdict || second.verdict === 'unexpected-screen') return verification;
|
|
1855
|
+
const was = String(verification?.observed?.to ?? '').slice(0, 8);
|
|
1856
|
+
const now = String(again?.hash ?? '').slice(0, 8);
|
|
1857
|
+
return {
|
|
1858
|
+
...verification,
|
|
1859
|
+
verdict: second.verdict,
|
|
1860
|
+
detail: `${second.detail} — read again after settling, because the first read said`
|
|
1861
|
+
+ ` "${verification.detail}"${was && now && was !== now ? ` (${was} → ${now})` : ''}`,
|
|
1862
|
+
// Both answers, so the log can count how often perception contradicts
|
|
1863
|
+
// itself here rather than only how often it halted a run.
|
|
1864
|
+
disagreed: { first: verification.verdict, second: second.verdict, from: was || null, to: now || null },
|
|
1865
|
+
lateArrival: again,
|
|
1866
|
+
};
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
/**
|
|
1870
|
+
* One more look before a wrong turn is allowed to stop the batch.
|
|
1871
|
+
*
|
|
1872
|
+
* The sibling of `confirmNoChange`, for the same reason and against the same
|
|
1873
|
+
* class of mistake: a verdict taken from a screen that was still arriving. The
|
|
1874
|
+
* reported symptom is that the gate is **non-deterministic** — a reporter
|
|
1875
|
+
* re-issued the identical call with no state change and it passed — and a gate
|
|
1876
|
+
* that fails once and passes on retry is flaky, not protective. Asked twice,
|
|
1877
|
+
* the perception disagrees with itself, and the first answer was never
|
|
1878
|
+
* evidence.
|
|
1879
|
+
*
|
|
1880
|
+
* This keeps the verify barrier rather than softening it. The barrier asks for
|
|
1881
|
+
* confirmed perception before acting on a screen an `unexpected-*` verdict has
|
|
1882
|
+
* touched; a halt on one premature read is not confirmed perception either.
|
|
1883
|
+
* Both directions now cost a second, settled read, and a wrong turn that is
|
|
1884
|
+
* real still halts — on better evidence than before.
|
|
1885
|
+
*
|
|
1886
|
+
* Deliberately not gated on a supervisor being configured. The escape hatch the
|
|
1887
|
+
* item proposed was a supervisor ruling, which `doctor` reports as
|
|
1888
|
+
* `none — not requested` on a default install, so that fix would have reached
|
|
1889
|
+
* only callers who opted in. A second read needs no model and no opt-in; a
|
|
1890
|
+
* supervisor, when there is one, still gets asked about what survives it.
|
|
1891
|
+
*/
|
|
1892
|
+
async function confirmWrongTurn(deviceQuery, verification, { udid, prediction, beforeScreen, step, options, stableMs, timeoutMs }) {
|
|
1893
|
+
if (verification?.verdict !== 'unexpected-screen' || !prediction || !beforeScreen?.hash) return verification;
|
|
1894
|
+
await api.waitFor(deviceQuery, {
|
|
1895
|
+
mode: 'settle', stableMs: 250, timeoutMs: LATE_CHANGE_MS, options,
|
|
1896
|
+
}).catch(() => null);
|
|
1897
|
+
const again = await api.screenIdentity(deviceQuery, { options, settleMs: stableMs, timeoutMs }).catch(() => null);
|
|
1898
|
+
// Nothing looked, so nothing was established and the first answer stands.
|
|
1899
|
+
// A failed read may never be read as agreement.
|
|
1900
|
+
if (!again?.hash) return verification;
|
|
1901
|
+
// `kind` is deliberately not passed: the transition classifier's reading
|
|
1902
|
+
// belongs to the moment of the first look, and it only ever decorates an
|
|
1903
|
+
// `ok`. Re-using it here would date-stamp this answer with that one.
|
|
1904
|
+
const second = graph.verdict({ udid, prediction, before: beforeScreen, after: again, action: step.action });
|
|
1905
|
+
return afterSecondLook(verification, second, again);
|
|
1906
|
+
}
|
|
1907
|
+
|
|
1778
1908
|
/**
|
|
1779
1909
|
* A control that changed state is not a step that did nothing.
|
|
1780
1910
|
*
|
|
@@ -2716,11 +2846,12 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2716
2846
|
return `pressed key ${step.value ?? step.code}`;
|
|
2717
2847
|
case 'launch': {
|
|
2718
2848
|
const bundleId = step.value ?? step.bundleId;
|
|
2719
|
-
const
|
|
2849
|
+
const shell = { waitedMs: 0, attempts: 0 };
|
|
2850
|
+
const started = await launchThroughShellCrash(udid, bundleId, {
|
|
2720
2851
|
args: step.args ?? [],
|
|
2721
2852
|
env: step.env ?? {},
|
|
2722
2853
|
terminateFirst: step.relaunch === true,
|
|
2723
|
-
});
|
|
2854
|
+
}, shell);
|
|
2724
2855
|
// Item 169: simctl returning ok means the process started, not that the
|
|
2725
2856
|
// app came forward, and the two came apart repeatedly on a loaded
|
|
2726
2857
|
// runner — leaving the device on the previous app under a step that
|
|
@@ -2760,6 +2891,10 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2760
2891
|
if (ctx.landing) ctx.landing.verdict = landed.verdict;
|
|
2761
2892
|
return `launched ${bundleId}${step.relaunch ? ' (relaunched)' : ''}`
|
|
2762
2893
|
+ (landed.verdict === 'fronted' ? ` (frontmost after ${landed.ms}ms)` : '')
|
|
2894
|
+
+ (shell.attempts > 1
|
|
2895
|
+
? ` [the guest's SpringBoard crashed and came back; launched again after ${shell.waitedMs}ms`
|
|
2896
|
+
+ `, on attempt ${shell.attempts}]`
|
|
2897
|
+
: '')
|
|
2763
2898
|
+ (retried
|
|
2764
2899
|
? ` [it did not front on the first attempt — ${frontmost.describeHeld(retried.held, started.pid)}`
|
|
2765
2900
|
+ `; a second launch fronted it. The launch is unreliable on this host.]`
|
|
@@ -2873,6 +3008,30 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2873
3008
|
// `direction` still wins, for a caller who knows better.
|
|
2874
3009
|
const asked = step.direction ? String(step.direction).toLowerCase() : null;
|
|
2875
3010
|
const max = Math.min(MAX_SCROLLS, step.maxScrolls ?? 6);
|
|
3011
|
+
// **The screen's size, in scope.** It was not, and that single omission is
|
|
3012
|
+
// the whole of item 190 and of the field report's Defect 2.
|
|
3013
|
+
//
|
|
3014
|
+
// `points` existed only inside `offsetSays`'s own destructure, so every
|
|
3015
|
+
// other reference to it — `inViewport`, and the message that reports a
|
|
3016
|
+
// target as out of view — was an undeclared identifier. `inViewport` runs
|
|
3017
|
+
// on **every successful locate**, so the moment this step found what it
|
|
3018
|
+
// was looking for it threw `ReferenceError: points is not defined`, the
|
|
3019
|
+
// loop's own `catch` swallowed it as "not found", and it scrolled on until
|
|
3020
|
+
// the budget ran out and it reported the target unreachable.
|
|
3021
|
+
//
|
|
3022
|
+
// So `scrollTo` could fail on a target that was on screen, could fail
|
|
3023
|
+
// after a scroll that had just brought the target into view, and could
|
|
3024
|
+
// burn its whole budget doing it — all three of the reported symptoms,
|
|
3025
|
+
// and all three are this. Measured from the loop, i is the iteration:
|
|
3026
|
+
//
|
|
3027
|
+
// i=0 locate threw: "Developer" is not on this screen (true, it was above)
|
|
3028
|
+
// i=1 locate threw: points is not defined (it had FOUND it)
|
|
3029
|
+
// i=2 locate threw: points is not defined (again)
|
|
3030
|
+
//
|
|
3031
|
+
// Optional chaining does not help: `points?.width` on an undeclared name
|
|
3032
|
+
// is still a ReferenceError, which is why nothing caught this by reading.
|
|
3033
|
+
const geo = await ctx.screen();
|
|
3034
|
+
const points = { width: geo.pointWidth, height: geo.pointHeight };
|
|
2876
3035
|
let dir = asked ?? 'down';
|
|
2877
3036
|
let reversed = false;
|
|
2878
3037
|
// Where the target is, when the tree knows. `null` means no evidence, and
|
|
@@ -2880,18 +3039,40 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2880
3039
|
// the top of a web page, which **triggers pull-to-refresh**, reloads the
|
|
2881
3040
|
// page and changes the screen hash — defeating the end-detection below
|
|
2882
3041
|
// and reading, from outside, as an endless loop. Observed live.
|
|
2883
|
-
|
|
2884
|
-
|
|
3042
|
+
//
|
|
3043
|
+
// **And it answers `null` far more often than this was written for.**
|
|
3044
|
+
// Measured on 2026-09-18, which is item 190: iOS puts only the *visible*
|
|
3045
|
+
// rows of a table in the accessibility tree. A scrolled-away row is not
|
|
3046
|
+
// there with an out-of-bounds coordinate — it is absent.
|
|
3047
|
+
//
|
|
3048
|
+
// Settings at the top, resolve("Developer") -> status=none, y=undefined
|
|
3049
|
+
// Settings at the bottom, resolve("Accessibility") -> status=none, y=undefined
|
|
3050
|
+
//
|
|
3051
|
+
// Zero elements whose label even contains the query, both times. So on a
|
|
3052
|
+
// native list this returns `null` in precisely the case it exists to
|
|
3053
|
+
// answer, and the y<0 / y>height reasoning below only ever fires on the
|
|
3054
|
+
// frameworks that do keep off-screen rows in the tree — React Native, and
|
|
3055
|
+
// the `y = -693` case this was written for.
|
|
3056
|
+
// One read answers both questions, so asking them costs what asking one
|
|
3057
|
+
// used to: which way is the target, and what is on screen right now.
|
|
3058
|
+
const lookAround = async () => {
|
|
2885
3059
|
try {
|
|
2886
|
-
const { entry
|
|
2887
|
-
const
|
|
3060
|
+
const { entry } = await api.readScreenWith(deviceQuery, { useOcr: false, options: ctx.options });
|
|
3061
|
+
const targets = entry.targets ?? [];
|
|
3062
|
+
// What is visible, as one comparable string. This is the end detector
|
|
3063
|
+
// now, and the frame hash is not — see `scrolledNowhere`.
|
|
3064
|
+
const signature = targets.map((t) => String(t.label ?? '')).filter(Boolean).sort().join('\u0000');
|
|
3065
|
+
if (asked) return { says: asked, signature };
|
|
3066
|
+
const hit = matching.resolve(targets, String(query));
|
|
2888
3067
|
const y = hit?.target?.y;
|
|
2889
|
-
|
|
2890
|
-
if (y
|
|
2891
|
-
if (y
|
|
2892
|
-
|
|
3068
|
+
let says = null;
|
|
3069
|
+
if (!Number.isFinite(y)) says = null;
|
|
3070
|
+
else if (y < 0) says = 'up';
|
|
3071
|
+
else if (y > (points?.height ?? Infinity)) says = 'down';
|
|
3072
|
+
else says = dir;
|
|
3073
|
+
return { says, signature };
|
|
2893
3074
|
} catch {
|
|
2894
|
-
return null;
|
|
3075
|
+
return { says: asked ?? null, signature: null };
|
|
2895
3076
|
}
|
|
2896
3077
|
};
|
|
2897
3078
|
/**
|
|
@@ -2924,25 +3105,57 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2924
3105
|
// rather than the act — reported as "after 1 scroll down" on a request
|
|
2925
3106
|
// for "up".
|
|
2926
3107
|
const scrolled = [];
|
|
3108
|
+
const arrivedHow = () => (scrolled.length
|
|
3109
|
+
? ` after ${scrolled.length} scroll${scrolled.length === 1 ? '' : 's'} `
|
|
3110
|
+
+ (new Set(scrolled).size === 1 ? scrolled[0] : scrolled.join(' then '))
|
|
3111
|
+
: ' already');
|
|
3112
|
+
/**
|
|
3113
|
+
* Is it here now? Asked without throwing, so a give-up path can use it.
|
|
3114
|
+
*
|
|
3115
|
+
* **A frame hash is a whole-screen measure and a strip is a small part of
|
|
3116
|
+
* the screen**, so "the frame did not change" is not "nothing moved".
|
|
3117
|
+
* Reported from the field: a horizontally-scrolling tab strip had the
|
|
3118
|
+
* target scrolled into view — the visible tabs demonstrably changed — and
|
|
3119
|
+
* this step reported `not reachable by scrolling` anyway, because the
|
|
3120
|
+
* whole-frame hash barely moved. A confident wrong *failure* after the
|
|
3121
|
+
* action succeeded, which costs what a silent success costs: the caller
|
|
3122
|
+
* is handed a false fact and a round trip.
|
|
3123
|
+
*
|
|
3124
|
+
* Same principle as the recall path (DEFERRED 184): a miss may not be the
|
|
3125
|
+
* final word until something has looked.
|
|
3126
|
+
*/
|
|
3127
|
+
const nowInView = async () => {
|
|
3128
|
+
try {
|
|
3129
|
+
const found = await api.locate(deviceQuery, query, { index: step.index, refresh: true });
|
|
3130
|
+
return inViewport(found.target) ? found : null;
|
|
3131
|
+
} catch {
|
|
3132
|
+
return null;
|
|
3133
|
+
}
|
|
3134
|
+
};
|
|
3135
|
+
// What the last look actually established, so the give-up message can say
|
|
3136
|
+
// what it knows instead of guessing. `absent` means the resolver did not
|
|
3137
|
+
// find it; `off-view` means it did and the target sits outside the
|
|
3138
|
+
// viewport — and in that case claiming it may not be in the tree is
|
|
3139
|
+
// flatly contradicted by the tree we just read.
|
|
3140
|
+
let lastMiss = null;
|
|
2927
3141
|
for (let i = 0; i <= max; i += 1) {
|
|
2928
3142
|
try {
|
|
2929
3143
|
const found = await api.locate(deviceQuery, query, { index: step.index, refresh: i > 0 });
|
|
2930
|
-
if (!inViewport(found.target))
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${how}`;
|
|
3144
|
+
if (!inViewport(found.target)) {
|
|
3145
|
+
lastMiss = 'off-view';
|
|
3146
|
+
throw new Error(
|
|
3147
|
+
`"${query}" is in the tree but not in view (at ${found.target.x},${found.target.y}`
|
|
3148
|
+
+ ` on a ${Math.round(points?.width ?? 0)}x${Math.round(points?.height ?? 0)}pt screen)`);
|
|
3149
|
+
}
|
|
3150
|
+
return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${arrivedHow()}`;
|
|
2938
3151
|
} catch (err) {
|
|
3152
|
+
if (lastMiss !== 'off-view') lastMiss = 'absent';
|
|
2939
3153
|
if (i === max) {
|
|
2940
3154
|
throw new Error(`scrolled ${dir} ${max}x without finding ${query}: ${err.message}`);
|
|
2941
3155
|
}
|
|
2942
3156
|
}
|
|
2943
|
-
const evidence = await
|
|
3157
|
+
const { says: evidence, signature: wasShowing } = await lookAround();
|
|
2944
3158
|
if (evidence) dir = evidence;
|
|
2945
|
-
const wasAt = await hashNow(deviceQuery, ctx.options);
|
|
2946
3159
|
scrolled.push(dir);
|
|
2947
3160
|
await runStep(deviceQuery, udid, { action: 'scroll', value: dir }, ctx);
|
|
2948
3161
|
// A scroll either moves immediately or not at all, so it does not need a
|
|
@@ -2956,17 +3169,64 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2956
3169
|
// page"*. Reverse once — the target may be behind us, and on a page
|
|
2957
3170
|
// whose fields never enter the tree there is no offset to follow — then
|
|
2958
3171
|
// stop rather than thrash.
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
3172
|
+
// **What "it stopped moving" is measured on, and the frame hash was the
|
|
3173
|
+
// wrong thing.** At the end of a list iOS rubber-bands: the content does
|
|
3174
|
+
// not advance and the pixels do, so a whole-frame comparison answers
|
|
3175
|
+
// "yes, it moved" on every attempt and the end is never detected. Seen
|
|
3176
|
+
// on 2026-09-18 — `scrolled down 6x without finding Accessibility` on a
|
|
3177
|
+
// list that had been against its bottom stop the whole time, 21.5s for
|
|
3178
|
+
// nothing.
|
|
3179
|
+
//
|
|
3180
|
+
// The visible labels do not bounce. If the same rows are on screen after
|
|
3181
|
+
// a scroll as before it, the scroll achieved nothing, whatever the
|
|
3182
|
+
// framebuffer did.
|
|
3183
|
+
const { signature: nowShowing } = await lookAround();
|
|
3184
|
+
if (wasShowing && nowShowing && wasShowing === nowShowing) {
|
|
3185
|
+
// Before believing the hash, look. See `nowInView` — an unchanged
|
|
3186
|
+
// whole-frame hash is weak evidence about a strip, and this step has
|
|
3187
|
+
// reported failure on a scroll that worked.
|
|
3188
|
+
const arrived = await nowInView();
|
|
3189
|
+
if (arrived) {
|
|
3190
|
+
return `"${arrived.target.label ?? query}" is in view at ${arrived.target.x},${arrived.target.y}`
|
|
3191
|
+
+ `${arrivedHow()} (the frame hash did not register the scroll — looked again rather than`
|
|
3192
|
+
+ ' reporting a failure the screen contradicts)';
|
|
3193
|
+
}
|
|
3194
|
+
// **A stall is evidence, and treating it as none is item 190.**
|
|
3195
|
+
//
|
|
3196
|
+
// This used to reverse only when the *tree* said where the target
|
|
3197
|
+
// was — and on a native list the tree never does, because scrolled-
|
|
3198
|
+
// away rows are not in it. So the common case was: no evidence, the
|
|
3199
|
+
// default direction, and if that direction happened to be the wrong
|
|
3200
|
+
// one the step burned its budget moving away from the target and then
|
|
3201
|
+
// refused, having never looked the other way. Field-measured at 12.6s
|
|
3202
|
+
// and 18.4s for zero progress, the largest single latency cost in
|
|
3203
|
+
// that round.
|
|
3204
|
+
//
|
|
3205
|
+
// Having scrolled and moved nothing, we know this direction is
|
|
3206
|
+
// exhausted. That is a fact about the screen rather than a guess, and
|
|
3207
|
+
// it is a different claim from the one the rule above was protecting
|
|
3208
|
+
// against: the danger there was *opening* with an unfounded "up",
|
|
3209
|
+
// which walks a web page to the top and pulls it to refresh. Here we
|
|
3210
|
+
// are at one end and the only place left to look is the other. Still
|
|
3211
|
+
// exactly once, still bounded by `max`.
|
|
3212
|
+
if (reversed) {
|
|
2965
3213
|
throw new Error(
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
3214
|
+
// Both ways, always — this branch is now only reached after a
|
|
3215
|
+
// reversal, so "there is no direction to try" (which it used to
|
|
3216
|
+
// say when the tree was silent) is no longer true and would be
|
|
3217
|
+
// the same kind of unchecked assertion as the sentence below it.
|
|
3218
|
+
`${query} is not reachable by scrolling: it stopped moving both ways`
|
|
3219
|
+
+ ` (${scrolled.join(', ')}) after ${i + 1} attempt(s).`
|
|
3220
|
+
// Say what was established, not what would be convenient. This
|
|
3221
|
+
// used to assert "It may not be in the accessibility tree at all"
|
|
3222
|
+
// in every case — a claim the function never checks, and one the
|
|
3223
|
+
// `off-view` branch has already disproved by reading the element
|
|
3224
|
+
// out of the tree a line earlier.
|
|
3225
|
+
+ (lastMiss === 'off-view'
|
|
3226
|
+
? ' It IS in the tree and outside the viewport, so this is a scrolling problem'
|
|
3227
|
+
+ ' rather than a perception one — try a different container, or aim at a coordinate.'
|
|
3228
|
+
: ' The resolver did not find it here either, so it may not be on this screen at all —'
|
|
3229
|
+
+ ' read the screen (sim_ui), or aim at a coordinate.'),
|
|
2970
3230
|
);
|
|
2971
3231
|
}
|
|
2972
3232
|
reversed = true;
|
package/src/cli.js
CHANGED
|
@@ -3,7 +3,7 @@ import fs from 'node:fs';
|
|
|
3
3
|
import os from 'node:os';
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import { runDaemon, DEFAULTS } from './daemon.js';
|
|
6
|
-
import { bootedDevices, capabilitiesFor, listDevices, PLATFORMS, resolveDevice,
|
|
6
|
+
import { bootedDevices, capabilitiesFor, listDevices, PLATFORMS, resolveDevice, screenshot, toolchainChecks } from './platform/index.js';
|
|
7
7
|
import * as actions from './actions.js';
|
|
8
8
|
import * as analyze from './analyze.js';
|
|
9
9
|
import * as api from './index.js';
|
|
@@ -12,11 +12,14 @@ import * as baseline from './baseline.js';
|
|
|
12
12
|
import * as metrics from './metrics.js';
|
|
13
13
|
import * as navigate from './navigate.js';
|
|
14
14
|
import * as wedge from './wedge.js';
|
|
15
|
+
|
|
15
16
|
import { decodePng } from './png.js';
|
|
16
17
|
import * as storage from './storage.js';
|
|
17
18
|
import * as store from './store.js';
|
|
18
19
|
import * as view from './view.js';
|
|
19
20
|
|
|
21
|
+
|
|
22
|
+
|
|
20
23
|
const USAGE = `simframe — always-warm iOS Simulator frames
|
|
21
24
|
|
|
22
25
|
simframe mcp run the MCP server on stdio (for agents)
|
|
@@ -421,31 +424,31 @@ async function main() {
|
|
|
421
424
|
case 'revive': {
|
|
422
425
|
const dev = await resolveDevice(device);
|
|
423
426
|
const say = (line) => { if (!flags.json) console.log(line); };
|
|
424
|
-
const steps = [];
|
|
425
|
-
const did = async (what, fn) => {
|
|
426
|
-
try { await fn(); steps.push({ step: what, ok: true }); say(` ok ${what}`); } catch (err) {
|
|
427
|
-
steps.push({ step: what, ok: false, error: err.message });
|
|
428
|
-
say(` .. ${what} — ${err.message.split('\n')[0]}`);
|
|
429
|
-
}
|
|
430
|
-
};
|
|
431
427
|
say(`reviving ${dev.name}`);
|
|
432
|
-
//
|
|
433
|
-
//
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
const
|
|
443
|
-
const
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
428
|
+
// The sequence itself lives in `wedge.js`, because `bench-hpi` needs the
|
|
429
|
+
// same recovery between passes and a second copy of it here is how this
|
|
430
|
+
// project has repeatedly ended up fixing one symptom in three places.
|
|
431
|
+
const revived = await wedge.revive(dev.udid, {
|
|
432
|
+
options,
|
|
433
|
+
device: dev,
|
|
434
|
+
onStep: ({ step, ok, error }) => say(ok ? ` ok ${step}` : ` .. ${step} — ${String(error).split('\n')[0]}`),
|
|
435
|
+
});
|
|
436
|
+
const { steps } = revived;
|
|
437
|
+
const diag = { verdict: revived.verdict };
|
|
438
|
+
const state = diag?.verdict?.state ?? 'read-failed';
|
|
439
|
+
const usable = !wedge.UNUSABLE.has(state);
|
|
440
|
+
const degraded = usable && state !== 'healthy';
|
|
441
|
+
emit(flags, { ok: usable, device: dev.udid, steps, verdict: diag?.verdict ?? null }, usable
|
|
442
|
+
? (degraded
|
|
443
|
+
? `\n${dev.name} is back and usable, but ${state}:\n ${diag.verdict.detail}`
|
|
444
|
+
+ '\nNot a failure — it taps and reads.'
|
|
445
|
+
: `\n${dev.name} is healthy again — ${diag.verdict.detail}`)
|
|
446
|
+
: `\n${dev.name} came back ${state}, which is not usable.`
|
|
447
|
+
+ `\n ${diag?.verdict?.detail ?? 'nothing could be read from it'}`
|
|
448
|
+
+ '\nA second revive sometimes clears it. If it does not, this is past what simframe'
|
|
449
|
+
+ ' can do — check Simulator.app is not showing an error, and see docs/DEFERRED.md'
|
|
450
|
+
+ ' items 95 and 183.');
|
|
451
|
+
if (!usable) process.exitCode = 1;
|
|
449
452
|
return;
|
|
450
453
|
}
|
|
451
454
|
|
|
@@ -461,7 +464,23 @@ async function main() {
|
|
|
461
464
|
'',
|
|
462
465
|
` frame seq ${r.frame?.seq ?? '-'}, ${r.frame?.ageMs ?? '-'}ms old, still for ${r.frame?.stableForMs ?? '-'}ms, ${r.frame?.size ?? '-'}`,
|
|
463
466
|
` elements ${r.elements.total} total — ${r.elements.ax} by tree, ${r.elements.ocr} by OCR, ${r.elements.fused} by both`,
|
|
464
|
-
|
|
467
|
+
// **No band here, and that is the second version of this fix.**
|
|
468
|
+
//
|
|
469
|
+
// A reporter saw `healthy` printed beside "measured healthy 0.66-0.93"
|
|
470
|
+
// over a reading of 0.567 and asked, reasonably, which to believe. The
|
|
471
|
+
// first fix widened the band to 0.57-0.93 — and the very next device
|
|
472
|
+
// read returned **0.471** on an ordinary Settings screen, which is the
|
|
473
|
+
// same contradiction one decimal place down. Any fixed band will be
|
|
474
|
+
// contradicted by the next screen, because fusion tracks how much of a
|
|
475
|
+
// screen's content is OCR-only and that is a property of the app.
|
|
476
|
+
//
|
|
477
|
+
// So the line prints the number and the one threshold the verdict
|
|
478
|
+
// actually turns on. The observed range lives in `docs/BENCHMARKS.md`,
|
|
479
|
+
// where it is evidence about screens rather than a standard a device is
|
|
480
|
+
// being held to. `HEALTHY_FUSION` is still printed by the `stale-frame`
|
|
481
|
+
// verdict, where a reading of 0.03 genuinely wants the contrast.
|
|
482
|
+
` fusion ${r.agreement ?? 'n/a'} of elements seen by both sensors`
|
|
483
|
+
+ ` — stale at or below ${wedge.DISAGREEMENT}, which is what this verdict turns on`,
|
|
465
484
|
` frontmost ${r.frontmost?.pid ?? 'unknown'}${r.frontmost?.title ? ` (${r.frontmost.title})` : ''}`,
|
|
466
485
|
...(r.verdict.revive ? ['', ' `simframe revive` is the recovery. Keep this output — item 173 needs it.'] : []),
|
|
467
486
|
]);
|
package/src/device-state.js
CHANGED
|
@@ -49,6 +49,18 @@ export const DEVICE_STATE = [
|
|
|
49
49
|
/never came to the front within \d+ms/i,
|
|
50
50
|
'a launched app never came to the front (the launch said so itself, by pid)',
|
|
51
51
|
],
|
|
52
|
+
// simctl itself stopped answering, and said so in simframe's own words: the
|
|
53
|
+
// process was killed at the timeout rather than refusing the request.
|
|
54
|
+
//
|
|
55
|
+
// Three runs of the 2026-09-17 bench died this way — 95.6 s each — and every
|
|
56
|
+
// one of them was written to `flows.jsonl` with `device_cause: null`, so the
|
|
57
|
+
// number CI gates on counted a simulator that had stopped answering as the
|
|
58
|
+
// code getting things wrong. That is the same fault item 173 recorded as
|
|
59
|
+
// fixed, still open for this signature because nothing here matched it.
|
|
60
|
+
[
|
|
61
|
+
/did not return within \d+s \(killed by simframe/i,
|
|
62
|
+
'simctl stopped answering and had to be killed at the timeout',
|
|
63
|
+
],
|
|
52
64
|
// Seen on the v0.14.3 bench run: `could not launch com.apple.Preferences:
|
|
53
65
|
// The system shell (SpringBoard:36454) probably crashed.` The guest's window
|
|
54
66
|
// server going down is the device, not the check, and nothing here matched it.
|
|
@@ -62,3 +74,28 @@ export const DEVICE_STATE = [
|
|
|
62
74
|
export function deviceCause(text) {
|
|
63
75
|
return DEVICE_STATE.find(([re]) => re.test(String(text ?? '')))?.[1] ?? null;
|
|
64
76
|
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The one device failure that is known to heal by itself.
|
|
80
|
+
*
|
|
81
|
+
* The guest's window server dies, the launch that was in flight fails, and
|
|
82
|
+
* SpringBoard comes back a few seconds later. Item 173 spent a dozen
|
|
83
|
+
* occurrences unable to observe it for exactly that reason — by the time
|
|
84
|
+
* anything looked, `diagnose` said `healthy`, fusion 0.857.
|
|
85
|
+
*
|
|
86
|
+
* Measured on `326464A4` on 2026-09-18, six occurrences in one run: waiting and
|
|
87
|
+
* launching again recovered **6 of 6**, in 5.7-7.9 s (median ~6.3 s). The
|
|
88
|
+
* remedy in use until now was `simframe revive`, a ~40 s device restart — six
|
|
89
|
+
* times the cost, for a fault that was already over.
|
|
90
|
+
*
|
|
91
|
+
* Separate from `deviceCause` on purpose. Every entry there says "this is the
|
|
92
|
+
* device"; this one says "and it will be back". `simctl did not return within
|
|
93
|
+
* 90s` is the device too and is deliberately NOT here: it was never once
|
|
94
|
+
* observed to recover, and a retry costs another 90 s to find that out.
|
|
95
|
+
*/
|
|
96
|
+
export const SHELL_CRASH = /system shell \(SpringBoard[^)]*\) probably crashed/i;
|
|
97
|
+
|
|
98
|
+
/** Did this failure come from the guest shell dying under us? */
|
|
99
|
+
export function shellCrashed(text) {
|
|
100
|
+
return SHELL_CRASH.test(String(text ?? ''));
|
|
101
|
+
}
|