simframe 0.14.0 → 0.14.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/native/simframed/Sources/PrivateAPI/AccessibilityBridge.swift +49 -2
- package/native/simframed/Sources/SimframeCore/Element.swift +19 -2
- package/package.json +1 -1
- package/scripts/analyse-escalations.mjs +89 -0
- package/scripts/analyse-routes.mjs +96 -0
- package/scripts/ci-memory.mjs +36 -5
- package/scripts/eval-fingerprint.mjs +152 -5
- package/src/actions.js +89 -7
- package/src/cli.js +39 -2
- package/src/index.js +15 -4
- package/src/input.js +33 -1
- package/src/matching.js +11 -1
- package/src/mcp.js +43 -2
- package/src/platform/ios.js +61 -9
- package/src/storage.js +55 -6
- package/src/view.js +1 -1
|
@@ -16,11 +16,20 @@ public struct AXNode: Sendable {
|
|
|
16
16
|
public var selected: Bool?
|
|
17
17
|
public var focused: Bool?
|
|
18
18
|
public var frame: CGRect
|
|
19
|
+
/// Where the control actuates, when the app says so.
|
|
20
|
+
///
|
|
21
|
+
/// Not the same as the centre of `frame`, and a switch is the proof: UIKit
|
|
22
|
+
/// publishes a row-wide frame for it, so the geometric centre is the label
|
|
23
|
+
/// and iOS does not actuate a switch from there. Measured — tapping the
|
|
24
|
+
/// centre flipped it 0 of 3 times and the control itself 3 of 3. UIKit has
|
|
25
|
+
/// `accessibilityActivationPoint` for exactly this question, and nil means
|
|
26
|
+
/// the app did not answer it, never a guess of ours.
|
|
27
|
+
public var activationPoint: CGPoint?
|
|
19
28
|
public var depth: Int
|
|
20
29
|
|
|
21
30
|
public init(role: String, subrole: String? = nil, label: String? = nil, value: String? = nil,
|
|
22
31
|
identifier: String? = nil, enabled: Bool? = nil, selected: Bool? = nil,
|
|
23
|
-
focused: Bool? = nil, frame: CGRect, depth: Int) {
|
|
32
|
+
focused: Bool? = nil, frame: CGRect, activationPoint: CGPoint? = nil, depth: Int) {
|
|
24
33
|
self.role = role
|
|
25
34
|
self.subrole = subrole
|
|
26
35
|
self.label = label
|
|
@@ -30,6 +39,7 @@ public struct AXNode: Sendable {
|
|
|
30
39
|
self.selected = selected
|
|
31
40
|
self.focused = focused
|
|
32
41
|
self.frame = frame
|
|
42
|
+
self.activationPoint = activationPoint
|
|
33
43
|
self.depth = depth
|
|
34
44
|
}
|
|
35
45
|
}
|
|
@@ -251,7 +261,8 @@ public final class AccessibilityBridge {
|
|
|
251
261
|
|
|
252
262
|
/// Everything scalar about a node, asked for in one go.
|
|
253
263
|
private static let batched = ["AXRole", "AXSubrole", "AXDescription", "AXValue",
|
|
254
|
-
"AXIdentifier", "AXEnabled", "AXSelected", "AXFocused"
|
|
264
|
+
"AXIdentifier", "AXEnabled", "AXSelected", "AXFocused",
|
|
265
|
+
"AXActivationPoint"]
|
|
255
266
|
|
|
256
267
|
private func node(from element: NSObject, depth: Int) -> AXNode {
|
|
257
268
|
// One bridge round trip for eight attributes, not eight.
|
|
@@ -283,6 +294,7 @@ public final class AccessibilityBridge {
|
|
|
283
294
|
selected: (value("AXSelected") as? NSNumber)?.boolValue,
|
|
284
295
|
focused: (value("AXFocused") as? NSNumber)?.boolValue,
|
|
285
296
|
frame: frame(of: element),
|
|
297
|
+
activationPoint: point(value("AXActivationPoint")) ?? activationPoint(of: element),
|
|
286
298
|
depth: depth)
|
|
287
299
|
}
|
|
288
300
|
|
|
@@ -310,6 +322,41 @@ public final class AccessibilityBridge {
|
|
|
310
322
|
return unsafeBitCast(imp, to: RectFn.self)(element, sel)
|
|
311
323
|
}
|
|
312
324
|
|
|
325
|
+
/// The activation point as the attribute bag reports it.
|
|
326
|
+
///
|
|
327
|
+
/// Defensive about the shape because this is a private bridge and the
|
|
328
|
+
/// answer arrives as whatever the translator chose: an NSValue wrapping a
|
|
329
|
+
/// CGPoint on one path, a stringified `{x, y}` on another. Anything it does
|
|
330
|
+
/// not recognise is nil, which costs nothing — the caller falls back to the
|
|
331
|
+
/// frame centre, which is what it did before this attribute was asked for.
|
|
332
|
+
private func point(_ value: Any?) -> CGPoint? {
|
|
333
|
+
switch value {
|
|
334
|
+
case let v as NSValue:
|
|
335
|
+
// `pointValue`, not `cgPointValue`: this daemon is a macOS binary
|
|
336
|
+
// driving a guest, and the iOS-only accessor does not exist here.
|
|
337
|
+
// NSPoint and CGPoint are the same type on 64-bit.
|
|
338
|
+
let p = v.pointValue
|
|
339
|
+
return p.x.isFinite && p.y.isFinite ? p : nil
|
|
340
|
+
case let s as String:
|
|
341
|
+
let parts = s.trimmingCharacters(in: CharacterSet(charactersIn: "{} "))
|
|
342
|
+
.split(separator: ",")
|
|
343
|
+
.compactMap { Double($0.trimmingCharacters(in: .whitespaces)) }
|
|
344
|
+
guard parts.count == 2 else { return nil }
|
|
345
|
+
return CGPoint(x: parts[0], y: parts[1])
|
|
346
|
+
default:
|
|
347
|
+
return nil
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/// The direct selector, for elements that answer it but do not batch it.
|
|
352
|
+
private func activationPoint(of element: NSObject) -> CGPoint? {
|
|
353
|
+
let sel = NSSelectorFromString("accessibilityActivationPoint")
|
|
354
|
+
guard element.responds(to: sel), let imp = element.method(for: sel) else { return nil }
|
|
355
|
+
typealias PointFn = @convention(c) (AnyObject, Selector) -> CGPoint
|
|
356
|
+
let p = unsafeBitCast(imp, to: PointFn.self)(element, sel)
|
|
357
|
+
return p.x.isFinite && p.y.isFinite ? p : nil
|
|
358
|
+
}
|
|
359
|
+
|
|
313
360
|
/// A value may be a string, a number, or something with no useful text.
|
|
314
361
|
private func string(_ value: Any?) -> String? {
|
|
315
362
|
switch value {
|
|
@@ -60,10 +60,20 @@ public struct Element: Sendable {
|
|
|
60
60
|
public var state: ElementState
|
|
61
61
|
public var source: ElementSource
|
|
62
62
|
public var confidence: Double
|
|
63
|
+
/// Where the control actuates, when the app publishes it.
|
|
64
|
+
///
|
|
65
|
+
/// Deliberately NOT folded into `center`. `center` is the element's place,
|
|
66
|
+
/// and two readings of one control are matched by how close their places
|
|
67
|
+
/// are — 12pt apart, per `SAME_CONTROL_POINTS` above the boundary. A switch
|
|
68
|
+
/// actuates at the far end of a row-wide frame, so making `center` mean
|
|
69
|
+
/// "where to tap" would move the tree's reading ~145pt away from OCR's and
|
|
70
|
+
/// stop the two collapsing into one control. Aiming and identity are
|
|
71
|
+
/// different questions; this is the answer to the first one only.
|
|
72
|
+
public var activationPoint: CGPoint?
|
|
63
73
|
|
|
64
74
|
public init(id: Int, frame: CGRect, role: String, label: String? = nil, value: String? = nil,
|
|
65
75
|
identifier: String? = nil, state: ElementState = ElementState(),
|
|
66
|
-
source: ElementSource, confidence: Double = 1) {
|
|
76
|
+
source: ElementSource, confidence: Double = 1, activationPoint: CGPoint? = nil) {
|
|
67
77
|
self.id = id
|
|
68
78
|
self.frame = frame
|
|
69
79
|
self.role = role
|
|
@@ -73,6 +83,7 @@ public struct Element: Sendable {
|
|
|
73
83
|
self.state = state
|
|
74
84
|
self.source = source
|
|
75
85
|
self.confidence = confidence
|
|
86
|
+
self.activationPoint = activationPoint
|
|
76
87
|
}
|
|
77
88
|
|
|
78
89
|
public var center: CGPoint { CGPoint(x: frame.midX, y: frame.midY) }
|
|
@@ -89,6 +100,11 @@ public struct Element: Sendable {
|
|
|
89
100
|
if let label { out["label"] = label }
|
|
90
101
|
if let value { out["value"] = value }
|
|
91
102
|
if let identifier { out["identifier"] = identifier }
|
|
103
|
+
// Only when the app answered. Absent means "no opinion", and the layer
|
|
104
|
+
// above then aims at the centre exactly as it always has.
|
|
105
|
+
if let activationPoint {
|
|
106
|
+
out["activationPoint"] = ["x": Int(activationPoint.x.rounded()), "y": Int(activationPoint.y.rounded())]
|
|
107
|
+
}
|
|
92
108
|
let state = state.json
|
|
93
109
|
if !state.isEmpty { out["state"] = state }
|
|
94
110
|
return out
|
|
@@ -112,7 +128,8 @@ public extension Element {
|
|
|
112
128
|
state: ElementState(enabled: node.enabled, selected: node.selected,
|
|
113
129
|
checked: nil, focused: node.focused),
|
|
114
130
|
source: .accessibility,
|
|
115
|
-
confidence: 1
|
|
131
|
+
confidence: 1,
|
|
132
|
+
activationPoint: node.activationPoint)
|
|
116
133
|
}
|
|
117
134
|
}
|
|
118
135
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "simframe",
|
|
3
|
-
"version": "0.14.
|
|
3
|
+
"version": "0.14.2",
|
|
4
4
|
"mcpName": "io.github.lvlrSajjad/simframe",
|
|
5
5
|
"description": "Always-warm iOS Simulator and Android emulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
|
|
6
6
|
"keywords": [
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Of the `verification_failed` escalations, how many happened on a
|
|
4
|
+
* `(screen, action)` the graph had already seen behave consistently?
|
|
5
|
+
* EXPERIMENTS §17. The answer was 3% and 0%, which is why this is kept: it is
|
|
6
|
+
* the measurement that stopped a phase being spent on remembering edge
|
|
7
|
+
* outcomes.
|
|
8
|
+
*
|
|
9
|
+
* Two thirds of those escalations are `no-visible-change`, and 83-100% of them
|
|
10
|
+
* are on a screen or an action the graph has never seen — memory cannot answer
|
|
11
|
+
* a question about a place it has never been.
|
|
12
|
+
*
|
|
13
|
+
* The graph is read as it is NOW, so an edge matched here may have been learned
|
|
14
|
+
* after the escalation. That makes the "addressable" figure an upper bound, and
|
|
15
|
+
* it is already near zero.
|
|
16
|
+
*
|
|
17
|
+
* Aggregate by design, and stricter than it looks: `detail` is app content, so
|
|
18
|
+
* verdicts are classified against a CLOSED vocabulary and the raw string is
|
|
19
|
+
* never printed. The first version of this printed the prefix and put a
|
|
20
|
+
* client's screen labels and a customer email into a terminal.
|
|
21
|
+
*
|
|
22
|
+
* Usage: node scripts/analyse-escalations.mjs <udid-prefix> [<udid-prefix>...] */
|
|
23
|
+
import fs from 'node:fs';
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
import os from 'node:os';
|
|
26
|
+
|
|
27
|
+
const root = process.env.SIMFRAME_HOME || path.join(os.homedir(), '.simframe');
|
|
28
|
+
|
|
29
|
+
for (const dev of process.argv.slice(2)) {
|
|
30
|
+
const dir = fs.readdirSync(root).find((d) => d.startsWith(dev));
|
|
31
|
+
if (!dir) continue;
|
|
32
|
+
const base = path.join(root, dir);
|
|
33
|
+
const gdir = path.join(base, 'graph');
|
|
34
|
+
if (!fs.existsSync(gdir)) continue;
|
|
35
|
+
|
|
36
|
+
const nodes = fs.readdirSync(gdir).filter((f) => f.endsWith('.json'))
|
|
37
|
+
.map((f) => { try { return JSON.parse(fs.readFileSync(path.join(gdir, f), 'utf8')); } catch { return null; } })
|
|
38
|
+
.filter(Boolean);
|
|
39
|
+
const byHash = new Map();
|
|
40
|
+
for (const n of nodes) { byHash.set(n.hash, n); for (const v of n.variants ?? []) if (!byHash.has(v.hash)) byHash.set(v.hash, n); }
|
|
41
|
+
|
|
42
|
+
const esc = [];
|
|
43
|
+
const p = path.join(base, 'escalations.jsonl');
|
|
44
|
+
if (!fs.existsSync(p)) continue;
|
|
45
|
+
for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
|
|
46
|
+
const t = line.trim(); if (!t) continue;
|
|
47
|
+
try { esc.push(JSON.parse(t)); } catch {}
|
|
48
|
+
}
|
|
49
|
+
const vf = esc.filter((e) => e.reason === 'verification_failed');
|
|
50
|
+
|
|
51
|
+
const bucket = { noScreen: 0, noEdge: 0, seenOnce: 0, consistent: 0, nondet: 0 };
|
|
52
|
+
const verdicts = new Map();
|
|
53
|
+
for (const e of vf) {
|
|
54
|
+
// Classify against a CLOSED vocabulary and never emit the raw text. The
|
|
55
|
+
// first version split on ':' and printed the prefix, which put a client's
|
|
56
|
+
// screen labels and an email address into a terminal. `detail` is app
|
|
57
|
+
// content; only its shape is ours to report.
|
|
58
|
+
const d = String(e.detail ?? '').toLowerCase();
|
|
59
|
+
const v = d.includes('no-visible-change') || d.includes('did not change') ? 'no-visible-change'
|
|
60
|
+
: d.includes('unexpected-screen') ? 'unexpected-screen'
|
|
61
|
+
: d.includes('did not settle') ? 'settle-timeout'
|
|
62
|
+
: d.includes('waited') ? 'wait-timeout'
|
|
63
|
+
: d.includes('matches') && d.includes('things') ? 'ambiguous'
|
|
64
|
+
: d.includes('numbered on a different') ? 'stale-ref'
|
|
65
|
+
: d.includes('disabled') ? 'disabled-control'
|
|
66
|
+
: d ? 'other' : '(none)';
|
|
67
|
+
verdicts.set(v, (verdicts.get(v) ?? 0) + 1);
|
|
68
|
+
const node = byHash.get(e.screen_fingerprint ?? '');
|
|
69
|
+
if (!node) { bucket.noScreen++; continue; }
|
|
70
|
+
const want = String(e.intent ?? '').toLowerCase();
|
|
71
|
+
const hit = (node.edges ?? []).find((x) => {
|
|
72
|
+
const sig = String(x.action ?? '').toLowerCase();
|
|
73
|
+
return want && (sig.endsWith(`:${want}`) || sig.includes(want));
|
|
74
|
+
});
|
|
75
|
+
if (!hit) { bucket.noEdge++; continue; }
|
|
76
|
+
if ((hit.changedOutcomes ?? 0) > 0) bucket.nondet++;
|
|
77
|
+
else if ((hit.count ?? 0) > 1) bucket.consistent++;
|
|
78
|
+
else bucket.seenOnce++;
|
|
79
|
+
}
|
|
80
|
+
const pct = (n) => (vf.length ? Math.round((100 * n) / vf.length) : 0);
|
|
81
|
+
console.log(`--- ${dev} ---`);
|
|
82
|
+
console.log(` verification_failed: ${vf.length}`);
|
|
83
|
+
console.log(` screen not in graph at all: ${bucket.noScreen} (${pct(bucket.noScreen)}%)`);
|
|
84
|
+
console.log(` screen known, this action never seen: ${bucket.noEdge} (${pct(bucket.noEdge)}%)`);
|
|
85
|
+
console.log(` edge seen exactly once: ${bucket.seenOnce} (${pct(bucket.seenOnce)}%)`);
|
|
86
|
+
console.log(` edge NONDETERMINISTIC (rightly asked): ${bucket.nondet} (${pct(bucket.nondet)}%)`);
|
|
87
|
+
console.log(` edge repeated and CONSISTENT: ${bucket.consistent} (${pct(bucket.consistent)}%) <- addressable`);
|
|
88
|
+
console.log(` verdict words: ${[...verdicts].sort((a,b)=>b[1]-a[1]).slice(0,6).map(([k,n])=>`${k}=${n}`).join(', ')}`);
|
|
89
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Would routing by measured time differ from routing by hop count, and by how
|
|
4
|
+
* much? EXPERIMENTS §16, kept runnable because the answer was "barely" and that
|
|
5
|
+
* is the kind of result that gets re-litigated by someone's intuition.
|
|
6
|
+
*
|
|
7
|
+
* `route()` in graph.js is a plain BFS over hop count, while `record()` has
|
|
8
|
+
* been storing a settle sample on nearly every edge all along (97% and 85% of
|
|
9
|
+
* edges on the two real-app graphs measured). This compares the two.
|
|
10
|
+
*
|
|
11
|
+
* Measured: the fastest route differs from the fewest-hops route on 9%, 0% and
|
|
12
|
+
* 7% of reachable pairs, saving a median of 1853ms, — and 615ms. Real, unused,
|
|
13
|
+
* and pointed at the wrong prize — it saves seconds, not round trips, and round
|
|
14
|
+
* trips are what the human-parity series exists to reduce.
|
|
15
|
+
*
|
|
16
|
+
* Output is aggregate by design: this reads a real device's memory of real
|
|
17
|
+
* third-party apps, so it prints counts and never a label, a screen or an app
|
|
18
|
+
* name. Same standing rule as scripts/phase17-corpus.mjs.
|
|
19
|
+
*
|
|
20
|
+
* Usage: node scripts/analyse-routes.mjs <udid-prefix> [<udid-prefix>...] */
|
|
21
|
+
import fs from 'node:fs';
|
|
22
|
+
import path from 'node:path';
|
|
23
|
+
import os from 'node:os';
|
|
24
|
+
|
|
25
|
+
const root = process.env.SIMFRAME_HOME || path.join(os.homedir(), '.simframe');
|
|
26
|
+
const median = (a) => { const s=[...a].sort((x,y)=>x-y); return s.length? (s.length%2? s[(s.length-1)/2] : (s[s.length/2-1]+s[s.length/2])/2) : null; };
|
|
27
|
+
|
|
28
|
+
for (const dev of process.argv.slice(2)) {
|
|
29
|
+
const dir = fs.readdirSync(root).find((d) => d.startsWith(dev));
|
|
30
|
+
if (!dir) continue;
|
|
31
|
+
const gdir = path.join(root, dir, 'graph');
|
|
32
|
+
if (!fs.existsSync(gdir)) continue;
|
|
33
|
+
const nodes = fs.readdirSync(gdir).filter(f=>f.endsWith('.json'))
|
|
34
|
+
.map(f => { try { return JSON.parse(fs.readFileSync(path.join(gdir,f),'utf8')); } catch { return null; } })
|
|
35
|
+
.filter(Boolean);
|
|
36
|
+
|
|
37
|
+
const byHash = new Map();
|
|
38
|
+
for (const n of nodes) { byHash.set(n.hash, n); for (const v of n.variants ?? []) if(!byHash.has(v.hash)) byHash.set(v.hash, n); }
|
|
39
|
+
const canonical = (h) => byHash.get(h)?.hash ?? h;
|
|
40
|
+
|
|
41
|
+
// Edge costs: the median settle actually measured on that edge.
|
|
42
|
+
const all = [];
|
|
43
|
+
for (const n of nodes) for (const e of n.edges ?? []) { const m = median(e.settles ?? []); if (m!=null) all.push(m); }
|
|
44
|
+
const fallback = median(all) ?? 500;
|
|
45
|
+
const cost = (e) => median(e.settles ?? []) ?? fallback;
|
|
46
|
+
|
|
47
|
+
// BFS exactly as graph.js does it: first path found, insertion order.
|
|
48
|
+
const bfs = (start, goal) => {
|
|
49
|
+
const seen = new Set([start]); const q = [{h:start, p:[]}];
|
|
50
|
+
while (q.length) { const {h,p} = q.shift();
|
|
51
|
+
if (p.length >= 12) continue;
|
|
52
|
+
for (const e of byHash.get(h)?.edges ?? []) {
|
|
53
|
+
const to = canonical(e.to); const next=[...p,e];
|
|
54
|
+
if (to === goal) return next;
|
|
55
|
+
if (seen.has(to)) continue; seen.add(to); q.push({h:to,p:next});
|
|
56
|
+
} }
|
|
57
|
+
return null;
|
|
58
|
+
};
|
|
59
|
+
// Dijkstra on measured time.
|
|
60
|
+
const fastest = (start, goal) => {
|
|
61
|
+
const dist = new Map([[start,0]]); const prev = new Map(); const done = new Set();
|
|
62
|
+
while (true) {
|
|
63
|
+
let cur=null, best=Infinity;
|
|
64
|
+
for (const [h,d] of dist) if (!done.has(h) && d<best) { best=d; cur=h; }
|
|
65
|
+
if (cur==null) break;
|
|
66
|
+
if (cur===goal) break;
|
|
67
|
+
done.add(cur);
|
|
68
|
+
for (const e of byHash.get(cur)?.edges ?? []) {
|
|
69
|
+
const to = canonical(e.to); const nd = best + cost(e);
|
|
70
|
+
if (nd < (dist.get(to) ?? Infinity)) { dist.set(to,nd); prev.set(to,[cur,e]); }
|
|
71
|
+
} }
|
|
72
|
+
if (!dist.has(goal)) return null;
|
|
73
|
+
const out=[]; let at=goal;
|
|
74
|
+
while (prev.has(at)) { const [from,e]=prev.get(at); out.unshift(e); at=from; }
|
|
75
|
+
return out;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const canon = [...new Set(nodes.map(n=>n.hash))];
|
|
79
|
+
let pairs=0, differ=0, saved=[], hopsUp=0;
|
|
80
|
+
for (const s of canon) for (const g of canon) {
|
|
81
|
+
if (s===g) continue;
|
|
82
|
+
const b = bfs(s,g); if (!b) continue;
|
|
83
|
+
const f = fastest(s,g); if (!f) continue;
|
|
84
|
+
pairs++;
|
|
85
|
+
const cb = b.reduce((t,e)=>t+cost(e),0), cf = f.reduce((t,e)=>t+cost(e),0);
|
|
86
|
+
if (cf < cb - 1) { differ++; saved.push(cb-cf); if (f.length > b.length) hopsUp++; }
|
|
87
|
+
}
|
|
88
|
+
saved.sort((a,b)=>a-b);
|
|
89
|
+
console.log(`--- ${dev} ---`);
|
|
90
|
+
console.log(` reachable pairs: ${pairs}`);
|
|
91
|
+
console.log(` where the fastest route differs from the fewest-hops route: ${differ} (${pairs?Math.round(100*differ/pairs):0}%)`);
|
|
92
|
+
if (saved.length) {
|
|
93
|
+
console.log(` time saved per such route — median ${Math.round(median(saved))}ms, max ${Math.round(saved[saved.length-1])}ms`);
|
|
94
|
+
console.log(` ...of those, routes that take MORE hops to be faster: ${hopsUp}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
package/scripts/ci-memory.mjs
CHANGED
|
@@ -225,10 +225,28 @@ function writeFlow(name, steps) {
|
|
|
225
225
|
return file;
|
|
226
226
|
}
|
|
227
227
|
|
|
228
|
-
// A closed loop:
|
|
229
|
-
// screens the graph can learn, and every pass starts where the last one
|
|
228
|
+
// A closed loop: launching an app puts it in front, home leaves it. Both ends
|
|
229
|
+
// are screens the graph can learn, and every pass starts where the last one
|
|
230
|
+
// ended.
|
|
231
|
+
//
|
|
232
|
+
// **It used to be `openUrl https://example.com`, and that was the bug.** Item
|
|
233
|
+
// 142 catalogued `simctl openurl` timing out on a loaded runner as a failure
|
|
234
|
+
// class, marked it "fixed — vehicle changed", and changed the vehicle in the
|
|
235
|
+
// *workflow's* step only. This loop was left on it, and so was the novel action
|
|
236
|
+
// below. That is the same class-versus-symptom error the `waitFor`/`assert`
|
|
237
|
+
// twin recorded: the fix went where the report pointed instead of everywhere
|
|
238
|
+
// the cause reached.
|
|
239
|
+
//
|
|
240
|
+
// It came back on 2026-09-14: passes 2 and 3 halted at step 0, the graph never
|
|
241
|
+
// got the chance to predict, and the failure read as "the outcome is predicted
|
|
242
|
+
// — pass 0", which names the graph for something Safari did.
|
|
243
|
+
//
|
|
244
|
+
// The replacement is the vehicle item 142 measured and proved for exactly this:
|
|
245
|
+
// **from inside an app, pressing home always changes the screen.** Settings is
|
|
246
|
+
// already installed everywhere this runs, the launch needs no network, and
|
|
247
|
+
// neither end depends on a browser cold-starting on a shared machine.
|
|
230
248
|
const LOOP = writeFlow('simframe-ci-loop.json', [
|
|
231
|
-
{
|
|
249
|
+
{ launch: { value: 'com.apple.Preferences', relaunch: true } },
|
|
232
250
|
{ button: 'home' },
|
|
233
251
|
]);
|
|
234
252
|
// Leaving whatever screen the map was read on.
|
|
@@ -537,10 +555,23 @@ if (novelRan && novelMoved) {
|
|
|
537
555
|
// so a run in which every pass failed to dispatch says nothing about
|
|
538
556
|
// prediction. It failed the build as `pass 0` while the real cause was a
|
|
539
557
|
// simctl launch timing out, three checks upstream.
|
|
540
|
-
|
|
541
|
-
|
|
558
|
+
//
|
|
559
|
+
// **The rule was right and the test of it was too coarse.** `anyPassRan` asks
|
|
560
|
+
// whether *any* pass ran, but prediction can only be observed on a pass AFTER
|
|
561
|
+
// the one that taught the edge — so pass 1 running is not enough. On
|
|
562
|
+
// 2026-09-14 pass 1 ran, passes 2 and 3 halted at step 0, and this reported
|
|
563
|
+
// `pass 0` as though the graph had declined to predict. Nothing had asked it
|
|
564
|
+
// to. Same sentence as the novel action three checks above: untested is not
|
|
565
|
+
// broken, and the guard has to test the pass the claim actually depends on.
|
|
566
|
+
const dispatched = (p) => Array.isArray(p.run?.results) && p.run.results.some((r) => r.ok !== false);
|
|
567
|
+
const laterPassRan = passes.slice(1).some(dispatched);
|
|
568
|
+
if (!passes.some(dispatched)) {
|
|
542
569
|
skip('and once the graph has seen it, the outcome is predicted',
|
|
543
570
|
'no pass dispatched a step, so the graph was never given anything to learn');
|
|
571
|
+
} else if (!laterPassRan) {
|
|
572
|
+
skip('and once the graph has seen it, the outcome is predicted',
|
|
573
|
+
`only the first pass dispatched a step (${passes.slice(1).map((p, i) => `pass ${i + 2}: [${p.verdicts.join(', ')}]`).join('; ')})`
|
|
574
|
+
+ ' — prediction is only observable on a pass after the one that taught the edge');
|
|
544
575
|
} else {
|
|
545
576
|
check(passes.some((p) => p.verdicts.includes('ok')),
|
|
546
577
|
'and once the graph has seen it, the outcome is predicted',
|
|
@@ -88,6 +88,8 @@ const readings = [];
|
|
|
88
88
|
const arrivalFailures = [];
|
|
89
89
|
/** Readings too bare to be a screen — see the guard where this is used. */
|
|
90
90
|
const sparseReadings = [];
|
|
91
|
+
/** `name|round` of every reading that stayed too sparse, so the report below can name the cause. */
|
|
92
|
+
const sparseAt = new Set();
|
|
91
93
|
/**
|
|
92
94
|
* Below this, a reading cannot distinguish its screen from any other bare one.
|
|
93
95
|
*
|
|
@@ -144,6 +146,14 @@ const save = (extra = {}) => {
|
|
|
144
146
|
* rather than hanging the job.
|
|
145
147
|
*/
|
|
146
148
|
const STALE_READ_RETRIES = 3;
|
|
149
|
+
/**
|
|
150
|
+
* How hard to try for a reading rich enough to tell its screen apart.
|
|
151
|
+
*
|
|
152
|
+
* Backed off rather than fixed, because the thing being waited for is a screen
|
|
153
|
+
* finishing its draw, and the runner that needs this is the slow one.
|
|
154
|
+
*/
|
|
155
|
+
const SPARSE_READ_RETRIES = 3;
|
|
156
|
+
const SPARSE_READ_WAIT_MS = 1500;
|
|
147
157
|
const STALE_READ_WAIT_MS = 1000;
|
|
148
158
|
|
|
149
159
|
/** When the steps that were supposed to change the screen finished. */
|
|
@@ -151,6 +161,52 @@ let navigatedAt = 0;
|
|
|
151
161
|
/** Readings that never got a frame newer than their own navigation. */
|
|
152
162
|
const staleReadings = [];
|
|
153
163
|
|
|
164
|
+
/**
|
|
165
|
+
* Visit every screen once before measuring, taking no readings.
|
|
166
|
+
*
|
|
167
|
+
* **The rounds are supposed to be repeat measurements of one thing.** They were
|
|
168
|
+
* not: round 1 systematically differed from rounds 2 and 3 whenever an app was
|
|
169
|
+
* cold, because an app that has just launched has not finished publishing its
|
|
170
|
+
* accessibility tree. Measured on CI — round 1 of `browser` read `e00725fce7`
|
|
171
|
+
* with **0 named** elements where rounds 2 and 3 read `9589eb2471` with **5**,
|
|
172
|
+
* while every other screen matched across all three. That is the eval measuring
|
|
173
|
+
* app cold-start, which it never set out to measure and does not report.
|
|
174
|
+
*
|
|
175
|
+
* It was invisible for a long time because something else was paying for it:
|
|
176
|
+
* the memory-layer step runs first on CI and drives the same apps for 504s,
|
|
177
|
+
* Safari included, so the eval always met a warm device. Sharding the job
|
|
178
|
+
* removed that neighbour and the dependency surfaced immediately. The defect
|
|
179
|
+
* was always here; the neighbour was hiding it.
|
|
180
|
+
*
|
|
181
|
+
* This is **not** the same as making the readings warm. Every reading is still
|
|
182
|
+
* taken with `fresh: true` against cold screen memory, which is what "cold"
|
|
183
|
+
* means in this harness — the graph must not have seen the screen before. What
|
|
184
|
+
* the warm-up removes is a variable about the *operating system* that the
|
|
185
|
+
* fingerprint has nothing to do with.
|
|
186
|
+
*
|
|
187
|
+
* Skippable with `--no-warmup`, because the comparison is the evidence: run it
|
|
188
|
+
* both ways to see whether round 1 still disagrees with its own repeats.
|
|
189
|
+
*/
|
|
190
|
+
async function warmUp() {
|
|
191
|
+
const started = Date.now();
|
|
192
|
+
console.log('warming: visiting each screen once, taking no readings');
|
|
193
|
+
for (const screen of tour) {
|
|
194
|
+
if (!screen.steps?.length) continue;
|
|
195
|
+
try {
|
|
196
|
+
await actions.runScript(device, { steps: screen.steps, verify: false });
|
|
197
|
+
} catch (err) {
|
|
198
|
+
// A warm-up failure is not a result. The measured rounds below will meet
|
|
199
|
+
// the same screen and fail there with the harness's own reporting, which
|
|
200
|
+
// says which screen and writes the readings out. Failing here would cost
|
|
201
|
+
// that and report a screen that was never measured.
|
|
202
|
+
console.log(` (warm-up could not reach "${screen.name}": ${err.message})`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
console.log(`warming: done in ${Math.round((Date.now() - started) / 1000)}s\n`);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (!process.argv.includes('--no-warmup')) await warmUp();
|
|
209
|
+
|
|
154
210
|
for (let round = 1; round <= rounds; round += 1) {
|
|
155
211
|
for (const screen of tour) {
|
|
156
212
|
if (screen.steps?.length) {
|
|
@@ -176,7 +232,30 @@ for (let round = 1; round <= rounds; round += 1) {
|
|
|
176
232
|
process.exit(1);
|
|
177
233
|
}
|
|
178
234
|
}
|
|
179
|
-
|
|
235
|
+
// A read can throw, and when it does the readings so far are the evidence.
|
|
236
|
+
//
|
|
237
|
+
// This file already says so — *"Everything read so far is written out
|
|
238
|
+
// first, because a failing run is the one whose evidence matters"* — and
|
|
239
|
+
// then left this call unguarded, so a runner whose OCR overran its budget
|
|
240
|
+
// produced a raw stack trace, no `--out` file, and nothing for
|
|
241
|
+
// `analyse-fingerprint.mjs` to read. Exactly the failure the `save()` above
|
|
242
|
+
// was written to prevent, one line away from it.
|
|
243
|
+
//
|
|
244
|
+
// Not retried here. The read budget is already the client's own give-up
|
|
245
|
+
// point, so a read that overran it is a statement about the machine, and
|
|
246
|
+
// the honest thing is to say which screen and how far the tour got.
|
|
247
|
+
let id;
|
|
248
|
+
try {
|
|
249
|
+
id = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
|
|
250
|
+
} catch (err) {
|
|
251
|
+
save({ abandonedAt: { screen: screen.name, round, error: err.message, phase: 'reading' } });
|
|
252
|
+
console.error(`\nFAIL round ${round}, "${screen.name}" could not be read: ${err.message}`);
|
|
253
|
+
console.error(`${readings.length} reading(s) taken before this were written out.`);
|
|
254
|
+
console.error('A read that overran its budget is a fact about the host, not about the');
|
|
255
|
+
console.error('fingerprint — the budget is already the client\'s own give-up point, so');
|
|
256
|
+
console.error('raising it would only move where the wait ends.');
|
|
257
|
+
process.exit(1);
|
|
258
|
+
}
|
|
180
259
|
// A reading off a frame older than the navigation is not a reading either.
|
|
181
260
|
//
|
|
182
261
|
// Same rule as the sparseness guard below, on the other axis, and it took a
|
|
@@ -227,13 +306,36 @@ for (let round = 1; round <= rounds; round += 1) {
|
|
|
227
306
|
// "untested is not passed" rule the memory harness learned. A screen that
|
|
228
307
|
// is genuinely this bare after a second look is a real finding.
|
|
229
308
|
if ((id.tokens ?? []).length < MIN_TOKENS_FOR_A_READING) {
|
|
309
|
+
// Read again, and keep the BEST reading rather than the last one.
|
|
310
|
+
//
|
|
311
|
+
// One retry was not enough and the failure it produced pointed at the
|
|
312
|
+
// wrong thing. On CI, `settings-general` read 4 tokens twice and then
|
|
313
|
+
// scored 1.00 against the *Settings root* — because the General screen's
|
|
314
|
+
// back button is labelled "Settings", so a General seen only down to its
|
|
315
|
+
// nav bar is structurally the same screen as the root. The harness
|
|
316
|
+
// reported it as a fingerprint that does not resemble its own screen. The
|
|
317
|
+
// fingerprint was fine; the look was too short. `settings-general` reads
|
|
318
|
+
// 5 tokens when it is read properly.
|
|
319
|
+
//
|
|
320
|
+
// Best-of, not last, because these reads are samples of a screen that is
|
|
321
|
+
// still drawing: a later read is usually richer but not reliably so, and
|
|
322
|
+
// throwing away a 5-token reading because the retry saw 4 would be the
|
|
323
|
+
// same bug with more steps.
|
|
230
324
|
const before = (id.tokens ?? []).length;
|
|
231
|
-
|
|
232
|
-
|
|
325
|
+
const seen = [before];
|
|
326
|
+
for (let attempt = 1; attempt <= SPARSE_READ_RETRIES; attempt += 1) {
|
|
327
|
+
await new Promise((r) => setTimeout(r, SPARSE_READ_WAIT_MS * attempt));
|
|
328
|
+
const again = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
|
|
329
|
+
seen.push((again.tokens ?? []).length);
|
|
330
|
+
if ((again.tokens ?? []).length > (id.tokens ?? []).length) id = again;
|
|
331
|
+
if ((id.tokens ?? []).length >= MIN_TOKENS_FOR_A_READING) break;
|
|
332
|
+
}
|
|
233
333
|
const after = (id.tokens ?? []).length;
|
|
234
|
-
console.log(` ("${screen.name}" read ${before} token(s) — too sparse to compare
|
|
334
|
+
console.log(` ("${screen.name}" read ${before} token(s) — too sparse to compare;`
|
|
335
|
+
+ ` read again: ${seen.slice(1).join(', ')} — kept ${after})`);
|
|
235
336
|
if (after < MIN_TOKENS_FOR_A_READING) {
|
|
236
|
-
sparseReadings.push(`${screen.name} round ${round}: ${after} token(s) after
|
|
337
|
+
sparseReadings.push(`${screen.name} round ${round}: ${after} token(s) after ${seen.length} reads`);
|
|
338
|
+
sparseAt.add(`${screen.name}|${round}`);
|
|
237
339
|
}
|
|
238
340
|
}
|
|
239
341
|
// Did we actually arrive? Two differently-named screens reading the same
|
|
@@ -419,6 +521,38 @@ if (strays.length) {
|
|
|
419
521
|
const matchNamed = match ? namedTokens(match.tokens) : [];
|
|
420
522
|
const collided = bestOther >= 0.99 && named.length === 0 && matchNamed.length === 0;
|
|
421
523
|
if (collided) collisions += 1;
|
|
524
|
+
// The third cause, and the one that produced this report on 2026-09-15.
|
|
525
|
+
//
|
|
526
|
+
// A reading that stayed under the token floor did not fail to resemble its
|
|
527
|
+
// screen — it never saw enough of the screen to resemble anything. On CI
|
|
528
|
+
// `settings-general` read 4 tokens and scored 1.00 against the Settings
|
|
529
|
+
// root, because the General screen's back button is labelled "Settings", so
|
|
530
|
+
// a General seen only down to its nav bar IS the root structurally. The
|
|
531
|
+
// report called that a wrong turn and sent the reader to fix the tour.
|
|
532
|
+
//
|
|
533
|
+
// Not folded into `collided` above: that one means the fingerprint had
|
|
534
|
+
// nothing to work with, which is this harness's subject. This means we did
|
|
535
|
+
// not look long enough, which is the harness's own fault and a different
|
|
536
|
+
// remedy.
|
|
537
|
+
//
|
|
538
|
+
// **And sparseness alone is not enough to claim it**, which this
|
|
539
|
+
// misdiagnosed once. A wrong turn that lands on a screen which legitimately
|
|
540
|
+
// reads sparse gets flagged sparse too, and was then reported as our
|
|
541
|
+
// instrument's fault rather than the tour's. Measured: `settings-general`
|
|
542
|
+
// r3 came back with tokens byte-identical to the Settings root, including
|
|
543
|
+
// `heading:nav-bar:"settings"` — a *heading*, which is the root's title,
|
|
544
|
+
// where General publishes a *button* with the same word. It was read on the
|
|
545
|
+
// wrong screen, and this called it an under-read.
|
|
546
|
+
//
|
|
547
|
+
// The discriminator is the screen's own other rounds: if `settings-general`
|
|
548
|
+
// read differently and richly in rounds 1 and 2, then the screen IS
|
|
549
|
+
// distinguishable and a round matching another screen exactly went
|
|
550
|
+
// somewhere else. Only when no round of this screen can tell itself apart
|
|
551
|
+
// is "we did not look long enough" the honest reading.
|
|
552
|
+
const distinguishable = (byName.get(reading.name) ?? [])
|
|
553
|
+
.some((o) => o !== reading && fingerprint.similarity(o.tokens, match?.tokens ?? []) < 0.99);
|
|
554
|
+
const underRead = sparseAt.has(`${reading.name}|${reading.round}`) && !distinguishable;
|
|
555
|
+
const wrongScreen = bestOther >= 0.99 && distinguishable;
|
|
422
556
|
console.error(` ${reading.name} r${reading.round}: own screen ${bestSelf.toFixed(2)}, `
|
|
423
557
|
+ `${match ? `${match.name} r${match.round}` : 'another screen'} ${bestOther.toFixed(2)} `
|
|
424
558
|
+ `(${reading.count} tokens, ${named.length} named, sources ${reading.sources.join('+') || 'none'})`);
|
|
@@ -426,6 +560,15 @@ if (strays.length) {
|
|
|
426
560
|
console.error(' ^ a COLLISION, not a wrong turn: neither reading carries a chrome');
|
|
427
561
|
console.error(' label, so both are structure with no name and the fingerprint has');
|
|
428
562
|
console.error(' nothing left to tell two list screens apart.');
|
|
563
|
+
} else if (wrongScreen) {
|
|
564
|
+
console.error(` ^ WRONG SCREEN: these tokens are ${match ? `identical to ${match.name}` : 'another screen'},`);
|
|
565
|
+
console.error(' and this screen reads differently in its other rounds — so it is');
|
|
566
|
+
console.error(' distinguishable and the tour was simply somewhere else. A tap that');
|
|
567
|
+
console.error(' missed, or a screen that went back before it was read.');
|
|
568
|
+
} else if (underRead) {
|
|
569
|
+
console.error(' ^ UNDER-READ, not a wrong turn: this reading stayed below the token');
|
|
570
|
+
console.error(' floor after every retry, so it never saw enough of its screen to');
|
|
571
|
+
console.error(' resemble one. Two screens read this thinly are the same screen.');
|
|
429
572
|
}
|
|
430
573
|
if (named.length) console.error(` names: ${named.map((t) => t.slice(t.indexOf('"'), t.lastIndexOf('"') + 1)).join(' ')}`);
|
|
431
574
|
}
|
|
@@ -433,6 +576,10 @@ if (strays.length) {
|
|
|
433
576
|
console.error(`\n${collisions} of ${strays.length} are fingerprint collisions. That is this harness's own subject,`);
|
|
434
577
|
console.error('not a tour fault: a reading whose chrome label went missing cannot establish');
|
|
435
578
|
console.error('identity, and comparing it as though it could is what produced the verdict above.');
|
|
579
|
+
} else if (strays.every((x) => sparseAt.has(`${x.reading.name}|${x.reading.round}`))) {
|
|
580
|
+
console.error('\nEvery stray above was UNDER-READ, so this says nothing about the tour or the');
|
|
581
|
+
console.error('fingerprint — the harness scored a look that was too short. The retries are in');
|
|
582
|
+
console.error('SPARSE_READ_RETRIES; a runner that needs more than they allow is the finding.');
|
|
436
583
|
} else {
|
|
437
584
|
console.error('\nThat is the tour going somewhere unintended, not the fingerprint drifting, and');
|
|
438
585
|
console.error('measuring it as either distribution poisons both ends. Fix the tour — a tap that');
|
package/src/actions.js
CHANGED
|
@@ -90,16 +90,49 @@ const ACTION_STEPS = new Set([
|
|
|
90
90
|
'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny', 'permission',
|
|
91
91
|
]);
|
|
92
92
|
|
|
93
|
+
/**
|
|
94
|
+
* Step keys that are really the MCP tool names, accepted as aliases.
|
|
95
|
+
*
|
|
96
|
+
* Two field reports guessed these independently and each wrong guess cost a
|
|
97
|
+
* round trip and aborted the rest of the batch. There is a `sim_wait` tool and
|
|
98
|
+
* a `sim_type_into` tool, so `wait` and `type_into` are what a caller reaches
|
|
99
|
+
* for inside `sim_do` — and the vocabulary answered "unknown step" without
|
|
100
|
+
* saying what the words are. A tool surface that names an action one way and
|
|
101
|
+
* accepts it another is charging the caller for our inconsistency.
|
|
102
|
+
*/
|
|
103
|
+
const STEP_ALIASES = Object.freeze({
|
|
104
|
+
wait: 'settle',
|
|
105
|
+
type_into: 'type',
|
|
106
|
+
typeInto: 'type',
|
|
107
|
+
scroll_to: 'scrollTo',
|
|
108
|
+
wait_for: 'waitFor',
|
|
109
|
+
waitfor: 'waitFor',
|
|
110
|
+
tap_at: 'tapAt',
|
|
111
|
+
open_url: 'openUrl',
|
|
112
|
+
assert_gone: 'assertGone',
|
|
113
|
+
assert_text: 'assertText',
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
/** Every step key `runStep` understands, for an error that can be acted on. */
|
|
117
|
+
export const STEP_KEYS = Object.freeze([
|
|
118
|
+
'tap', 'tapAt', 'type', 'paste', 'clear', 'swipe', 'scroll', 'scrollTo',
|
|
119
|
+
'button', 'key', 'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny',
|
|
120
|
+
'permission', 'settle', 'waitFor', 'waitText', 'assert', 'assertGone',
|
|
121
|
+
'assertText', 'visible', 'gone', 'enabled', 'disabled', 'value', 'look',
|
|
122
|
+
'seek', 'sweep', 'pause',
|
|
123
|
+
]);
|
|
124
|
+
|
|
93
125
|
/** Accept both `{tap: "Save"}` shorthand and `{action: "tap", target: "Save"}`. */
|
|
94
126
|
export function normalizeStep(raw) {
|
|
95
|
-
|
|
96
|
-
if (raw
|
|
127
|
+
const canonical = (a) => STEP_ALIASES[a] ?? a;
|
|
128
|
+
if (typeof raw === 'string') return { action: canonical(raw) };
|
|
129
|
+
if (raw.action) return { ...raw, action: canonical(raw.action) };
|
|
97
130
|
const [key] = Object.keys(raw);
|
|
98
131
|
if (!key) throw new Error('empty step');
|
|
99
132
|
// Siblings like timeoutMs sit alongside the shorthand key and must survive.
|
|
100
133
|
const { [key]: value, ...rest } = raw;
|
|
101
134
|
const inline = value && typeof value === 'object' && !Array.isArray(value) ? value : { value };
|
|
102
|
-
const step = { ...rest, ...inline, action: key };
|
|
135
|
+
const step = { ...rest, ...inline, action: canonical(key) };
|
|
103
136
|
// Drop keys that are present but undefined. `simframe tap X` used to pass
|
|
104
137
|
// `index: undefined`, which survived here and then crashed the signature
|
|
105
138
|
// builder before the tap was ever sent.
|
|
@@ -2486,11 +2519,47 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2486
2519
|
return null;
|
|
2487
2520
|
}
|
|
2488
2521
|
};
|
|
2522
|
+
/**
|
|
2523
|
+
* Is this target actually on the screen?
|
|
2524
|
+
*
|
|
2525
|
+
* Resolving is not the same as being in view, and this returned `"X" is
|
|
2526
|
+
* in view at 201,-35 after 1 scroll down` — 35pt ABOVE the viewport, so
|
|
2527
|
+
* the tap that followed missed. Reported from the field. The tree carries
|
|
2528
|
+
* scrolled-away rows with out-of-bounds coordinates, which is exactly the
|
|
2529
|
+
* case `scrollTo` exists to resolve, so claiming success on one is the
|
|
2530
|
+
* one answer it must never give.
|
|
2531
|
+
*
|
|
2532
|
+
* Generous rather than strict: any overlap with the viewport counts, so a
|
|
2533
|
+
* row half over an edge is still reachable and still reported. Only a
|
|
2534
|
+
* target entirely outside keeps the search going.
|
|
2535
|
+
*/
|
|
2536
|
+
const inViewport = (t) => {
|
|
2537
|
+
const w = points?.width;
|
|
2538
|
+
const h = points?.height;
|
|
2539
|
+
if (!Number.isFinite(w) || !Number.isFinite(h)) return true;
|
|
2540
|
+
const f = t?.frame;
|
|
2541
|
+
if (f && Number.isFinite(f.y) && Number.isFinite(f.height)) {
|
|
2542
|
+
return f.y + f.height > 0 && f.y < h && f.x + (f.width ?? 0) > 0 && f.x < w;
|
|
2543
|
+
}
|
|
2544
|
+
return Number.isFinite(t?.y) && t.y >= 0 && t.y <= h
|
|
2545
|
+
&& Number.isFinite(t?.x) && t.x >= 0 && t.x <= w;
|
|
2546
|
+
};
|
|
2547
|
+
// The direction actually scrolled, which is not always the one asked for:
|
|
2548
|
+
// `offsetSays()` may reverse it, and the message used to name the request
|
|
2549
|
+
// rather than the act — reported as "after 1 scroll down" on a request
|
|
2550
|
+
// for "up".
|
|
2551
|
+
const scrolled = [];
|
|
2489
2552
|
for (let i = 0; i <= max; i += 1) {
|
|
2490
2553
|
try {
|
|
2491
2554
|
const found = await api.locate(deviceQuery, query, { index: step.index, refresh: i > 0 });
|
|
2492
|
-
|
|
2493
|
-
|
|
2555
|
+
if (!inViewport(found.target)) throw new Error(
|
|
2556
|
+
`"${query}" is in the tree but not in view (at ${found.target.x},${found.target.y}`
|
|
2557
|
+
+ ` on a ${Math.round(points?.width ?? 0)}x${Math.round(points?.height ?? 0)}pt screen)`);
|
|
2558
|
+
const how = scrolled.length
|
|
2559
|
+
? ` after ${scrolled.length} scroll${scrolled.length === 1 ? '' : 's'} `
|
|
2560
|
+
+ (new Set(scrolled).size === 1 ? scrolled[0] : scrolled.join(' then '))
|
|
2561
|
+
: ' already';
|
|
2562
|
+
return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${how}`;
|
|
2494
2563
|
} catch (err) {
|
|
2495
2564
|
if (i === max) {
|
|
2496
2565
|
throw new Error(`scrolled ${dir} ${max}x without finding ${query}: ${err.message}`);
|
|
@@ -2499,6 +2568,7 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2499
2568
|
const evidence = await offsetSays();
|
|
2500
2569
|
if (evidence) dir = evidence;
|
|
2501
2570
|
const wasAt = await hashNow(deviceQuery, ctx.options);
|
|
2571
|
+
scrolled.push(dir);
|
|
2502
2572
|
await runStep(deviceQuery, udid, { action: 'scroll', value: dir }, ctx);
|
|
2503
2573
|
// A scroll either moves immediately or not at all, so it does not need a
|
|
2504
2574
|
// transition's budget. Six iterations at 2,500ms was most of why this
|
|
@@ -2763,8 +2833,20 @@ async function runStep(deviceQuery, udid, step, ctx) {
|
|
|
2763
2833
|
await sleep(ms);
|
|
2764
2834
|
return `paused ${ms}ms`;
|
|
2765
2835
|
}
|
|
2766
|
-
default:
|
|
2767
|
-
|
|
2836
|
+
default: {
|
|
2837
|
+
// Say what the words are. "unknown step" on its own sent two testers
|
|
2838
|
+
// guessing, and a guess costs a round trip and the rest of the batch.
|
|
2839
|
+
const near = STEP_KEYS.filter((k) => {
|
|
2840
|
+
const a = String(step.action ?? '').toLowerCase().replace(/[_-]/g, '');
|
|
2841
|
+
const b = k.toLowerCase();
|
|
2842
|
+
return a && (b.startsWith(a) || a.startsWith(b) || b.includes(a));
|
|
2843
|
+
}).slice(0, 3);
|
|
2844
|
+
throw new Error(
|
|
2845
|
+
`unknown step "${step.action}"`
|
|
2846
|
+
+ (near.length ? ` — did you mean ${near.map((k) => `"${k}"`).join(' or ')}?` : '')
|
|
2847
|
+
+ ` Valid steps: ${STEP_KEYS.join(', ')}.`,
|
|
2848
|
+
);
|
|
2849
|
+
}
|
|
2768
2850
|
}
|
|
2769
2851
|
}
|
|
2770
2852
|
|
package/src/cli.js
CHANGED
|
@@ -34,7 +34,7 @@ const USAGE = `simframe — always-warm iOS Simulator frames
|
|
|
34
34
|
simframe tap <selector> tap #3, "Save", or @120,400
|
|
35
35
|
simframe do <script.json> run a scripted flow (see below)
|
|
36
36
|
simframe screens [device] list screens this device has learned
|
|
37
|
-
simframe storage [bundle-id]
|
|
37
|
+
simframe storage [bundle-id] [--device=<name|udid>] what the app saved (no boot needed)
|
|
38
38
|
simframe goto <screen> walk to a known screen through known steps
|
|
39
39
|
simframe flow save <name> <script.json> run a flow and save it if every step verifies
|
|
40
40
|
simframe flow run <name> replay a saved flow
|
|
@@ -867,7 +867,44 @@ async function main() {
|
|
|
867
867
|
// whole value of this command is that it answers on a device that is not
|
|
868
868
|
// running — measured: simctl itself cannot, on this Xcode. Routing it
|
|
869
869
|
// through the daemon would throw that away for no gain.
|
|
870
|
-
|
|
870
|
+
// Resolving a device here must NOT require one to be running, and it did.
|
|
871
|
+
//
|
|
872
|
+
// The whole claim of this command is that it answers before a boot, and
|
|
873
|
+
// `resolveDevice(undefined)` resolves against *booted* devices — so the
|
|
874
|
+
// documented form, `simframe storage <bundle-id>`, failed with "no booted
|
|
875
|
+
// simulator" at exactly the feature's headline use case. Reported by an
|
|
876
|
+
// external tester, who found it worked only when a device was named,
|
|
877
|
+
// which the help line did not mention either.
|
|
878
|
+
//
|
|
879
|
+
// Named: resolve as usual, booted or not. Unnamed: prefer a booted device
|
|
880
|
+
// when there is one, otherwise the only device that has app containers —
|
|
881
|
+
// and when that is ambiguous, say so and name the candidates rather than
|
|
882
|
+
// complaining about a boot nobody needs.
|
|
883
|
+
const named = flags.device ?? flags.udid ?? process.env.SIMFRAME_DEVICE;
|
|
884
|
+
let device;
|
|
885
|
+
if (named) {
|
|
886
|
+
device = await resolveDevice(String(named));
|
|
887
|
+
} else {
|
|
888
|
+
device = await resolveDevice(undefined).catch(() => null);
|
|
889
|
+
if (!device) {
|
|
890
|
+
const all = await listDevices({ all: true });
|
|
891
|
+
const withApps = [];
|
|
892
|
+
for (const d of all) {
|
|
893
|
+
const apps = await storage.apps(d.udid).catch(() => []);
|
|
894
|
+
if (apps.length) withApps.push({ device: d, apps: apps.length });
|
|
895
|
+
}
|
|
896
|
+
if (withApps.length === 1) {
|
|
897
|
+
device = withApps[0].device;
|
|
898
|
+
} else {
|
|
899
|
+
throw new Error(
|
|
900
|
+
withApps.length
|
|
901
|
+
? 'storage reads a device that does not have to be running, so name which one:\n'
|
|
902
|
+
+ withApps.map((w) => ` --device=${w.device.udid} ${w.device.name} (${w.apps} app(s))`).join('\n')
|
|
903
|
+
: 'no device on this host has any app data to read',
|
|
904
|
+
);
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
}
|
|
871
908
|
const [bundleId] = positional;
|
|
872
909
|
if (!bundleId) {
|
|
873
910
|
const list = await storage.apps(device.udid);
|
package/src/index.js
CHANGED
|
@@ -1489,6 +1489,17 @@ export function sensorMode(options) {
|
|
|
1489
1489
|
return raw === 'ax-first' || raw === 'axfirst' ? 'ax-first' : 'full';
|
|
1490
1490
|
}
|
|
1491
1491
|
|
|
1492
|
+
/**
|
|
1493
|
+
* What to call an element in a "Visible:" list.
|
|
1494
|
+
*
|
|
1495
|
+
* The label, or the accessibility identifier when there is no label. Both are
|
|
1496
|
+
* now matchable, and the list had been filtered to `t.label` alone — so an
|
|
1497
|
+
* element `sim_ui` had just printed by identifier was missing from the list of
|
|
1498
|
+
* what is on screen, in the same reply that refused to resolve it. Reported
|
|
1499
|
+
* from the field, three times in one session.
|
|
1500
|
+
*/
|
|
1501
|
+
const nameFor = (t) => t.label || t.identifier || null;
|
|
1502
|
+
|
|
1492
1503
|
export async function locate(deviceQuery, query, opts = {}) {
|
|
1493
1504
|
if (sensorMode(opts.options) !== 'ax-first' || opts.useOcr === false || opts.escalated) {
|
|
1494
1505
|
return locateWith(deviceQuery, query, opts);
|
|
@@ -1701,8 +1712,8 @@ async function locateWith(
|
|
|
1701
1712
|
// here undoes every guard above — it has no off-screen filter and no
|
|
1702
1713
|
// coverage weighting, and it is what returned a scrolled-away list row for
|
|
1703
1714
|
// "back". "Not found" is the correct answer.
|
|
1704
|
-
const visible = entry.targets.filter((t) => t
|
|
1705
|
-
const sample = visible.slice(0, 12).map((t) => t.
|
|
1715
|
+
const visible = entry.targets.filter((t) => nameFor(t) && !regions.offViewport(t, points));
|
|
1716
|
+
const sample = visible.slice(0, 12).map((t) => nameFor(t).slice(0, 24)).join(', ');
|
|
1706
1717
|
// "Not on this screen" and "not in view" are different answers, and giving
|
|
1707
1718
|
// the first for the second cost a reported 15 seconds: a `waitFor REVIEW`
|
|
1708
1719
|
// burned its whole timeout while REVIEW sat one scroll below the fold, and
|
|
@@ -1734,8 +1745,8 @@ async function locateWith(
|
|
|
1734
1745
|
const candidates = screenmap.rank(entry, query);
|
|
1735
1746
|
const target = index != null ? candidates[index] : candidates[0];
|
|
1736
1747
|
if (!target) {
|
|
1737
|
-
const visible = entry.targets.filter((t) => t
|
|
1738
|
-
const shown = visible.slice(0, 12).map((t) => t
|
|
1748
|
+
const visible = entry.targets.filter((t) => nameFor(t));
|
|
1749
|
+
const shown = visible.slice(0, 12).map((t) => nameFor(t));
|
|
1739
1750
|
// Say when the list is cut. A field report found `"Work Orders" is not on
|
|
1740
1751
|
// this screen. Visible: …` on a screen whose own element map, three lines
|
|
1741
1752
|
// below in the same reply, listed `#25 text 200,836 Work Orders` — it was
|
package/src/input.js
CHANGED
|
@@ -254,6 +254,14 @@ export function elementToNode(e) {
|
|
|
254
254
|
selected: e.state?.selected ?? null,
|
|
255
255
|
focused: e.state?.focused ?? null,
|
|
256
256
|
frame: e.frame ?? null,
|
|
257
|
+
// Where the control actuates, when the app publishes it. See `centerOf`.
|
|
258
|
+
//
|
|
259
|
+
// Added here as well as in `normalizeNode`, and the comment four lines up
|
|
260
|
+
// is the reason: this is the converter that actually runs, and a field set
|
|
261
|
+
// only in the idb fallback is a field that is never set. That is exactly
|
|
262
|
+
// how AXSelected and AXFocused came to be batched by the daemon for four
|
|
263
|
+
// versions and dropped on the way in.
|
|
264
|
+
activationPoint: e.activationPoint ?? null,
|
|
257
265
|
raw: e,
|
|
258
266
|
};
|
|
259
267
|
}
|
|
@@ -292,6 +300,8 @@ function normalizeNode(node) {
|
|
|
292
300
|
height: frame.height ?? frame.Height ?? 0,
|
|
293
301
|
}
|
|
294
302
|
: null,
|
|
303
|
+
// Where the control actuates, when the app says so. See `centerOf`.
|
|
304
|
+
activationPoint: node.activationPoint ?? node.AXActivationPoint ?? null,
|
|
295
305
|
raw: node,
|
|
296
306
|
};
|
|
297
307
|
}
|
|
@@ -329,11 +339,33 @@ export function matchElement(nodes, query, { index } = {}) {
|
|
|
329
339
|
throw new Error(`no element matching "${query}" is on screen`);
|
|
330
340
|
}
|
|
331
341
|
|
|
342
|
+
/**
|
|
343
|
+
* Where to aim at this element.
|
|
344
|
+
*
|
|
345
|
+
* The geometric centre, unless the app has published somewhere better. UIKit
|
|
346
|
+
* exposes `accessibilityActivationPoint` for controls whose hit target is not
|
|
347
|
+
* the middle of what they publish as their frame, and a switch is the case that
|
|
348
|
+
* forced this: it reports a row-wide frame, so the centre is the *label*, and
|
|
349
|
+
* iOS does not actuate a switch from there. Measured on Settings →
|
|
350
|
+
* Accessibility → Hover Text — the frame centre flipped it **0 of 3** times and
|
|
351
|
+
* the control itself **3 of 3**, while the step reported `[no visible change]`
|
|
352
|
+
* and was telling the truth.
|
|
353
|
+
*
|
|
354
|
+
* The app's answer is preferred whenever it lands inside the frame. Outside it
|
|
355
|
+
* is not trusted: a point that is not on the element is not a better guess than
|
|
356
|
+
* the middle of one, and this runs on the tap path, where a wrong guess is the
|
|
357
|
+
* one thing that does damage.
|
|
358
|
+
*/
|
|
332
359
|
export function centerOf(node) {
|
|
333
|
-
|
|
360
|
+
const middle = {
|
|
334
361
|
x: Math.round(node.frame.x + node.frame.width / 2),
|
|
335
362
|
y: Math.round(node.frame.y + node.frame.height / 2),
|
|
336
363
|
};
|
|
364
|
+
const p = node.activationPoint;
|
|
365
|
+
if (!p || !Number.isFinite(p.x) || !Number.isFinite(p.y)) return middle;
|
|
366
|
+
const f = node.frame;
|
|
367
|
+
const inside = p.x >= f.x && p.x <= f.x + f.width && p.y >= f.y && p.y <= f.y + f.height;
|
|
368
|
+
return inside ? { x: Math.round(p.x), y: Math.round(p.y) } : middle;
|
|
337
369
|
}
|
|
338
370
|
|
|
339
371
|
export async function tapPoint(udid, x, y, { durationMs } = {}) {
|
package/src/matching.js
CHANGED
|
@@ -157,7 +157,17 @@ export function rank(targets, intent, { screen } = {}) {
|
|
|
157
157
|
|
|
158
158
|
const scored = [];
|
|
159
159
|
for (const t of visible) {
|
|
160
|
-
|
|
160
|
+
// The accessibility identifier is a name a caller can legitimately write,
|
|
161
|
+
// and this list did not contain it.
|
|
162
|
+
//
|
|
163
|
+
// `sim_ui` prints elements BY identifier — in React Native a `testID`
|
|
164
|
+
// becomes one, so it is most interactive controls in an RN app — and the
|
|
165
|
+
// resolver then rejected that exact string, in the same response that had
|
|
166
|
+
// just printed it, with the identifier also absent from the "Visible:"
|
|
167
|
+
// list. An external tester reproduced it three times and called it the
|
|
168
|
+
// single biggest friction of their session. `screenmap.rank` had learned
|
|
169
|
+
// this; this ranker, which is the one `resolve()` uses, had not.
|
|
170
|
+
const names = [t.label, t.identifier, ...(t.aliases ?? [])].filter(Boolean);
|
|
161
171
|
let base = 0;
|
|
162
172
|
let matched = null;
|
|
163
173
|
for (const name of names) {
|
package/src/mcp.js
CHANGED
|
@@ -405,7 +405,8 @@ const TOOLS = [
|
|
|
405
405
|
'What the app saved, as text: its UserDefaults and (for React Native) its AsyncStorage, read straight out of'
|
|
406
406
|
+ ' the data container. sim_ui says what is drawn; sim_storage says what the app believes — use it when the'
|
|
407
407
|
+ ' screen and the behaviour disagree, or to check a value without driving the UI to it.'
|
|
408
|
-
+ ' Works on a device that is NOT running, so it can answer before anything is booted
|
|
408
|
+
+ ' Works on a device that is NOT running, so it can answer before anything is booted —'
|
|
409
|
+
+ ' pass device when nothing is booted and the host has more than one simulator with data.'
|
|
409
410
|
+ ' Call with no bundleId to list the apps that have a container (match filters that list).',
|
|
410
411
|
inputSchema: {
|
|
411
412
|
type: 'object',
|
|
@@ -772,6 +773,23 @@ async function look(target, args, options) {
|
|
|
772
773
|
lines.push(`could not crop that region (${cropped.note}) — this is the whole screen`);
|
|
773
774
|
}
|
|
774
775
|
}
|
|
776
|
+
// Say what space the image is in, and the factor.
|
|
777
|
+
//
|
|
778
|
+
// A field report lost time to this and had to derive the number from
|
|
779
|
+
// landmarks: `sim_ui` and `sim_tap` speak 402x874 POINTS, the returned image
|
|
780
|
+
// is 322x700 PIXELS, and `simctl io screenshot` is a third space again at 3x.
|
|
781
|
+
// Reading a coordinate off this image and feeding it to a tap is silently
|
|
782
|
+
// wrong by 1.248x — silently, because nothing in the reply relates the two.
|
|
783
|
+
// The header already prints the frame size; it just never said what it was
|
|
784
|
+
// relative to.
|
|
785
|
+
const geometry = await api.screenIdentity(target, { options, confirmNovel: false }).catch(() => null);
|
|
786
|
+
const pts = geometry?.points;
|
|
787
|
+
if (pts?.width && res.state?.width) {
|
|
788
|
+
const factor = res.state.width / pts.width;
|
|
789
|
+
lines.push(`this image is ${res.state.width}x${res.state.height} px = `
|
|
790
|
+
+ `${Math.round(pts.width)}x${Math.round(pts.height)}pt — divide image coordinates by `
|
|
791
|
+
+ `${factor.toFixed(3)} before tapping (sim_tap and sim_ui speak points)`);
|
|
792
|
+
}
|
|
775
793
|
return { content: [text(lines.filter(Boolean).join('\n')), image(png)] };
|
|
776
794
|
}
|
|
777
795
|
|
|
@@ -1200,7 +1218,30 @@ function listStateDirs() {
|
|
|
1200
1218
|
* the host filesystem. That is what the backend does.
|
|
1201
1219
|
*/
|
|
1202
1220
|
async function appStorage({ bundleId, match: query, device } = {}) {
|
|
1203
|
-
|
|
1221
|
+
// Same correction as the CLI: this must not require a booted device, because
|
|
1222
|
+
// answering before a boot is the point of the tool. A bare call used to fail
|
|
1223
|
+
// with "no booted simulator" — reported from the field — so an unnamed device
|
|
1224
|
+
// falls back to the only one that has app data before it gives up.
|
|
1225
|
+
let resolved = device
|
|
1226
|
+
? await resolveDevice(String(device))
|
|
1227
|
+
: await resolveDevice(undefined).catch(() => null);
|
|
1228
|
+
if (!resolved) {
|
|
1229
|
+
const all = await listDevices({ all: true });
|
|
1230
|
+
const withApps = [];
|
|
1231
|
+
for (const d of all) {
|
|
1232
|
+
const apps = await storage.apps(d.udid).catch(() => []);
|
|
1233
|
+
if (apps.length) withApps.push({ device: d, apps: apps.length });
|
|
1234
|
+
}
|
|
1235
|
+
if (withApps.length === 1) resolved = withApps[0].device;
|
|
1236
|
+
else {
|
|
1237
|
+
throw new Error(
|
|
1238
|
+
withApps.length
|
|
1239
|
+
? 'storage reads a device that does not have to be running, so say which one: '
|
|
1240
|
+
+ withApps.map((w) => `${w.device.name} (${w.device.udid}, ${w.apps} app(s))`).join('; ')
|
|
1241
|
+
: 'no device on this host has any app data to read',
|
|
1242
|
+
);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1204
1245
|
if (!bundleId) {
|
|
1205
1246
|
const list = await storage.apps(resolved.udid);
|
|
1206
1247
|
const needle = query ? String(query).toLowerCase() : null;
|
package/src/platform/ios.js
CHANGED
|
@@ -13,6 +13,46 @@ import * as plist from './plist.js';
|
|
|
13
13
|
|
|
14
14
|
const run = promisify(execFile);
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* How long a `simctl` verb may take before we stop waiting.
|
|
18
|
+
*
|
|
19
|
+
* **It was 20s, and that was below what a loaded runner actually needs.** Item
|
|
20
|
+
* 142 recorded `simctl launch` "taking 47-55s" on a hosted runner and filed it
|
|
21
|
+
* under a boot that had not finished; the launches were real and the budget was
|
|
22
|
+
* simply shorter than they were. The same 20s sat on `openurl`, which is the
|
|
23
|
+
* failure class that item listed four runs of, and on `terminate`. One number,
|
|
24
|
+
* three symptoms, and every one of them read as the command refusing rather
|
|
25
|
+
* than as us leaving.
|
|
26
|
+
*
|
|
27
|
+
* 90s is chosen against that measurement — comfortably past the observed 55s
|
|
28
|
+
* worst case — and not for feel. A `simctl` verb that has not returned in
|
|
29
|
+
* ninety seconds is genuinely wrong, and says so below instead of being
|
|
30
|
+
* indistinguishable from a rejection.
|
|
31
|
+
*/
|
|
32
|
+
const SIMCTL_TIMEOUT_MS = 90_000;
|
|
33
|
+
|
|
34
|
+
/** A local file decode, which owes nothing to device latency. */
|
|
35
|
+
const PLUTIL_TIMEOUT_MS = 20_000;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* What actually went wrong, including the case that has been invisible.
|
|
39
|
+
*
|
|
40
|
+
* On a timeout `execFile` kills the child, so `stderr` is EMPTY and `message`
|
|
41
|
+
* is the bare "Command failed: xcrun simctl ..." — which reads exactly like
|
|
42
|
+
* simctl rejecting the request. Three separate investigations have started from
|
|
43
|
+
* that sentence and gone looking for a broken device. The timeout has to name
|
|
44
|
+
* itself, or the next one starts in the same wrong place.
|
|
45
|
+
*/
|
|
46
|
+
function simctlFailure(err, what) {
|
|
47
|
+
const detail = (err.stderr || '').trim().split('\n').filter(Boolean).pop();
|
|
48
|
+
if (detail) return `${what}: ${detail}`;
|
|
49
|
+
if (err.killed || err.signal === 'SIGTERM') {
|
|
50
|
+
return `${what}: simctl did not return within ${Math.round(SIMCTL_TIMEOUT_MS / 1000)}s`
|
|
51
|
+
+ ' (killed by simframe, not refused by simctl — the host is loaded or the device is not answering)';
|
|
52
|
+
}
|
|
53
|
+
return `${what}: ${err.message}`;
|
|
54
|
+
}
|
|
55
|
+
|
|
16
56
|
// `simctl list` costs ~130ms, which would otherwise dominate every warm read,
|
|
17
57
|
// so the parsed list is cached for a few seconds.
|
|
18
58
|
const DEVICE_CACHE_MS = 4000;
|
|
@@ -240,24 +280,33 @@ async function launchApp(udid, bundleId, { args = [], env = {}, terminateFirst =
|
|
|
240
280
|
for (const [k, v] of Object.entries(env)) childEnv[`SIMCTL_CHILD_${k}`] = String(v);
|
|
241
281
|
try {
|
|
242
282
|
await run('xcrun', ['simctl', 'launch', udid, bundleId, ...args.map(String)], {
|
|
243
|
-
timeout:
|
|
283
|
+
timeout: SIMCTL_TIMEOUT_MS,
|
|
244
284
|
env: childEnv,
|
|
245
285
|
});
|
|
246
286
|
} catch (err) {
|
|
247
287
|
// execFile's message is just "Command failed: ..." with simctl's actual
|
|
248
288
|
// complaint left in stderr. A CI run failed here and said nothing about
|
|
249
289
|
// why, which is the same sin as a silent fallback.
|
|
250
|
-
|
|
251
|
-
throw new Error(detail ? `could not launch ${bundleId}: ${detail}` : `could not launch ${bundleId}: ${err.message}`);
|
|
290
|
+
throw new Error(simctlFailure(err, `could not launch ${bundleId}`));
|
|
252
291
|
}
|
|
253
292
|
}
|
|
254
293
|
|
|
255
294
|
async function terminateApp(udid, bundleId) {
|
|
256
|
-
|
|
295
|
+
try {
|
|
296
|
+
await run('xcrun', ['simctl', 'terminate', udid, bundleId], { timeout: SIMCTL_TIMEOUT_MS });
|
|
297
|
+
} catch (err) {
|
|
298
|
+
throw new Error(simctlFailure(err, `could not terminate ${bundleId}`));
|
|
299
|
+
}
|
|
257
300
|
}
|
|
258
301
|
|
|
259
302
|
async function openUrl(udid, url) {
|
|
260
|
-
|
|
303
|
+
// The same budget and the same reporting as `launch`, because it was the same
|
|
304
|
+
// 20s and it is the failure class item 142 counted four runs of.
|
|
305
|
+
try {
|
|
306
|
+
await run('xcrun', ['simctl', 'openurl', udid, url], { timeout: SIMCTL_TIMEOUT_MS });
|
|
307
|
+
} catch (err) {
|
|
308
|
+
throw new Error(simctlFailure(err, 'could not open the url'));
|
|
309
|
+
}
|
|
261
310
|
}
|
|
262
311
|
|
|
263
312
|
/**
|
|
@@ -305,10 +354,9 @@ async function setPermission(udid, action, service, bundleId) {
|
|
|
305
354
|
const args = ['simctl', 'privacy', udid, verb, service];
|
|
306
355
|
if (bundleId) args.push(bundleId);
|
|
307
356
|
try {
|
|
308
|
-
await run('xcrun', args, { timeout:
|
|
357
|
+
await run('xcrun', args, { timeout: SIMCTL_TIMEOUT_MS });
|
|
309
358
|
} catch (err) {
|
|
310
|
-
|
|
311
|
-
throw new Error(`could not ${verb} ${service}: ${detail || err.message}`);
|
|
359
|
+
throw new Error(simctlFailure(err, `could not ${verb} ${service}`));
|
|
312
360
|
}
|
|
313
361
|
return `${verb === 'reset' ? 'reset' : verb + 'ed'} ${service}${bundleId ? ` for ${bundleId}` : ''}`;
|
|
314
362
|
}
|
|
@@ -422,8 +470,12 @@ async function appContainer(udid, bundleId) {
|
|
|
422
470
|
* the note at the top of plist.js.
|
|
423
471
|
*/
|
|
424
472
|
async function readPropertyList(file) {
|
|
473
|
+
// Its own budget, deliberately not the simctl one. This reads a local file
|
|
474
|
+
// and never speaks to a device, so it has none of the latency the simctl
|
|
475
|
+
// budget exists to absorb — and a plist that takes twenty seconds to decode
|
|
476
|
+
// is a problem worth hearing about promptly.
|
|
425
477
|
const { stdout } = await run('plutil', ['-convert', 'xml1', '-o', '-', file], {
|
|
426
|
-
timeout:
|
|
478
|
+
timeout: PLUTIL_TIMEOUT_MS,
|
|
427
479
|
maxBuffer: 64 * 1024 * 1024,
|
|
428
480
|
});
|
|
429
481
|
return plist.parse(stdout);
|
package/src/storage.js
CHANGED
|
@@ -39,6 +39,33 @@ export const VALUE_PREVIEW_BYTES = 4096;
|
|
|
39
39
|
const ASYNC_STORAGE_DIR = 'RCTAsyncLocalStorage_V1';
|
|
40
40
|
const ASYNC_STORAGE_MANIFEST = 'manifest.json';
|
|
41
41
|
|
|
42
|
+
/**
|
|
43
|
+
* Where AsyncStorage actually lives, newest layout first.
|
|
44
|
+
*
|
|
45
|
+
* **`Documents/` alone was wrong, and wrong in the worst available way.** That
|
|
46
|
+
* is the *legacy* React Native location. The community package every current RN
|
|
47
|
+
* app uses — `@react-native-async-storage/async-storage` — writes to
|
|
48
|
+
* `Library/Application Support/<bundle-id>/`, and on a real app measured by an
|
|
49
|
+
* external tester the `Documents/` path **did not exist at all**. So
|
|
50
|
+
* `readAsyncStorage` returned null, `format()` omitted the section, and the
|
|
51
|
+
* output read as "this app has no AsyncStorage" while 25 keys sat on disk,
|
|
52
|
+
* including a 1.5 MB MobX-State-Tree root store.
|
|
53
|
+
*
|
|
54
|
+
* Their diagnosis was exact: *"The decoder is correct; only the path is wrong."*
|
|
55
|
+
* Which makes this the precise class of confident wrong answer the whole feature
|
|
56
|
+
* exists to prevent, shipped inside it.
|
|
57
|
+
*
|
|
58
|
+
* Both are tried because both are real — an older app still writes to
|
|
59
|
+
* `Documents/` — and every path looked in is reported, because "found nothing"
|
|
60
|
+
* and "did not look there" are different facts and only one of them is about
|
|
61
|
+
* the app.
|
|
62
|
+
*/
|
|
63
|
+
const asyncStorageDirs = (container, bundleId) => [
|
|
64
|
+
path.join(container, 'Library', 'Application Support', String(bundleId ?? ''), ASYNC_STORAGE_DIR),
|
|
65
|
+
path.join(container, 'Library', 'Application Support', ASYNC_STORAGE_DIR),
|
|
66
|
+
path.join(container, 'Documents', ASYNC_STORAGE_DIR),
|
|
67
|
+
];
|
|
68
|
+
|
|
42
69
|
/** Files that are plainly a store but that nothing here can decode yet. */
|
|
43
70
|
const OPAQUE_STORES = /\.(sqlite3?|db|realm|leveldb|mmkv)$/i;
|
|
44
71
|
|
|
@@ -59,10 +86,13 @@ export async function apps(udid) {
|
|
|
59
86
|
* reporting it as one would be the exact class of wrong answer this feature
|
|
60
87
|
* exists to stop.
|
|
61
88
|
*/
|
|
62
|
-
export function readAsyncStorage(container) {
|
|
63
|
-
const
|
|
89
|
+
export function readAsyncStorage(container, bundleId) {
|
|
90
|
+
const looked = asyncStorageDirs(container, bundleId);
|
|
91
|
+
const dir = looked.find((d) => fs.existsSync(path.join(d, ASYNC_STORAGE_MANIFEST)));
|
|
92
|
+
// The paths travel with the miss. A reader told only "no AsyncStorage" cannot
|
|
93
|
+
// tell a bare app from a store we failed to find, and the second is ours.
|
|
94
|
+
if (!dir) return { missing: true, looked: looked.map((d) => path.relative(container, d)) };
|
|
64
95
|
const manifestPath = path.join(dir, ASYNC_STORAGE_MANIFEST);
|
|
65
|
-
if (!fs.existsSync(manifestPath)) return null;
|
|
66
96
|
const manifest = readJson(manifestPath);
|
|
67
97
|
const entries = [];
|
|
68
98
|
for (const [key, inline] of Object.entries(manifest)) {
|
|
@@ -151,9 +181,15 @@ export function typeOf(value) {
|
|
|
151
181
|
export async function read(udid, bundleId) {
|
|
152
182
|
const container = await platform.appContainer(udid, bundleId);
|
|
153
183
|
const stores = await readPreferences(udid, container, bundleId);
|
|
154
|
-
const async_ = readAsyncStorage(container);
|
|
155
|
-
if (async_) stores.push(async_);
|
|
156
|
-
return {
|
|
184
|
+
const async_ = readAsyncStorage(container, bundleId);
|
|
185
|
+
if (async_ && !async_.missing) stores.push(async_);
|
|
186
|
+
return {
|
|
187
|
+
bundleId,
|
|
188
|
+
container,
|
|
189
|
+
stores,
|
|
190
|
+
opaque: opaqueStores(container),
|
|
191
|
+
asyncStorageMissing: async_?.missing ? async_.looked : null,
|
|
192
|
+
};
|
|
157
193
|
}
|
|
158
194
|
|
|
159
195
|
/** One value, rendered for reading, saying so whenever it is not the whole thing. */
|
|
@@ -186,11 +222,24 @@ export function format(result) {
|
|
|
186
222
|
lines.push(` ${renderValue(e.value).split('\n').join('\n ')}`);
|
|
187
223
|
}
|
|
188
224
|
}
|
|
225
|
+
// Say where we looked and did not find it. See `asyncStorageDirs`.
|
|
226
|
+
if (result.asyncStorageMissing) {
|
|
227
|
+
lines.push('');
|
|
228
|
+
lines.push(' no AsyncStorage found. Looked in:');
|
|
229
|
+
for (const d of result.asyncStorageMissing) lines.push(` ${d}`);
|
|
230
|
+
lines.push(' (an app that does not use AsyncStorage will have none of these)');
|
|
231
|
+
}
|
|
189
232
|
if (result.opaque?.length) {
|
|
190
233
|
lines.push('');
|
|
191
234
|
lines.push(` ${result.opaque.length} store(s) present that this cannot decode yet:`);
|
|
192
235
|
for (const o of result.opaque) lines.push(` ${o.file} ${o.bytes} bytes`);
|
|
193
236
|
}
|
|
237
|
+
// Asked for by the external tester, and they were right to: mid-session the
|
|
238
|
+
// app restored a session from the Keychain and walked past its own login
|
|
239
|
+
// screen, so "logged out" as read from storage was not the whole truth.
|
|
240
|
+
// Saying what is NOT readable sets expectations that silence does not.
|
|
241
|
+
lines.push('');
|
|
242
|
+
lines.push(' Keychain is not readable from here — auth state may differ from what is above.');
|
|
194
243
|
return lines.join('\n');
|
|
195
244
|
}
|
|
196
245
|
|
package/src/view.js
CHANGED
|
@@ -579,7 +579,7 @@ export async function screenMap(deviceQuery, {
|
|
|
579
579
|
const anonymous = rows.filter(unaddressable).length;
|
|
580
580
|
const unnamed = anonymous
|
|
581
581
|
? `${anonymous} on-screen control(s) have no accessibility label — they are listed with their`
|
|
582
|
-
+ ' coordinates and can be tapped by
|
|
582
|
+
+ ' coordinates and can be tapped by #ref, or by point with tapAt. If what you are'
|
|
583
583
|
+ ' looking for is not in the list either, the app has views that were never declared'
|
|
584
584
|
+ ' accessible and only a screenshot will find those.'
|
|
585
585
|
: null;
|