simframe 0.4.2 → 0.6.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +334 -85
- package/native/simframed/Package.swift +16 -0
- package/native/simframed/Sources/PrivateAPI/AccessibilityBridge.swift +379 -0
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +523 -0
- package/native/simframed/Sources/PrivateAPI/HIDKeyboard.swift +70 -0
- package/native/simframed/Sources/PrivateAPI/IndigoHID.swift +121 -0
- package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +149 -0
- package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +112 -0
- package/native/simframed/Sources/SimframeCore/Bitmap.swift +61 -0
- package/native/simframed/Sources/SimframeCore/ControlSocket.swift +122 -0
- package/native/simframed/Sources/SimframeCore/CoreGraphicsScaler.swift +70 -0
- package/native/simframed/Sources/SimframeCore/Element.swift +148 -0
- package/native/simframed/Sources/SimframeCore/FrameStore.swift +303 -0
- package/native/simframed/Sources/SimframeCore/Hashing.swift +119 -0
- package/native/simframed/Sources/SimframeCore/Motion.swift +431 -0
- package/native/simframed/Sources/SimframeCore/PNGWriter.swift +40 -0
- package/native/simframed/Sources/SimframeCore/VisionOCR.swift +75 -0
- package/native/simframed/Sources/simframed/main.swift +485 -0
- package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +270 -0
- package/package.json +12 -4
- package/scripts/bench-flow.mjs +54 -0
- package/scripts/bench.sh +98 -0
- package/scripts/check-package.mjs +99 -0
- package/scripts/ci-memory.mjs +416 -0
- package/scripts/eval-fingerprint.mjs +192 -0
- package/scripts/smoke.mjs +76 -0
- package/scripts/sync-server-version.mjs +39 -0
- package/scripts/verify-baseline.mjs +65 -0
- package/skills/simframe/SKILL.md +173 -0
- package/src/actions.js +264 -18
- package/src/cli.js +561 -89
- package/src/control.js +77 -0
- package/src/daemon.js +8 -1
- package/src/engine.js +99 -0
- package/src/fingerprint.js +183 -0
- package/src/graph.js +411 -0
- package/src/index.js +351 -24
- package/src/input.js +179 -2
- package/src/matching.js +265 -0
- package/src/mcp.js +425 -112
- package/src/navigate.js +120 -0
- package/src/refs.js +141 -0
- package/src/regions.js +267 -0
- package/src/screenmap.js +119 -22
- package/src/simctl.js +74 -5
- package/src/store.js +8 -0
- package/src/view.js +342 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
import PrivateAPI
|
|
3
|
+
@testable import SimframeCore
|
|
4
|
+
|
|
5
|
+
/// These run without a simulator, against StubPlatform, so the pure logic is
|
|
6
|
+
/// testable on any machine — including CI, which has no booted device.
|
|
7
|
+
final class HashingTests: XCTestCase {
|
|
8
|
+
|
|
9
|
+
private func solid(_ w: Int, _ h: Int, _ v: UInt8) -> Bitmap {
|
|
10
|
+
var data = [UInt8](repeating: 0, count: w * h * 4)
|
|
11
|
+
for i in stride(from: 0, to: data.count, by: 4) {
|
|
12
|
+
data[i] = v; data[i + 1] = v; data[i + 2] = v; data[i + 3] = 255
|
|
13
|
+
}
|
|
14
|
+
return Bitmap(width: w, height: h, data: data)
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
private func split(_ w: Int, _ h: Int, left: UInt8, right: UInt8) -> Bitmap {
|
|
18
|
+
var data = [UInt8](repeating: 0, count: w * h * 4)
|
|
19
|
+
for y in 0..<h {
|
|
20
|
+
for x in 0..<w {
|
|
21
|
+
let v = x < w / 2 ? left : right
|
|
22
|
+
let i = (y * w + x) * 4
|
|
23
|
+
data[i] = v; data[i + 1] = v; data[i + 2] = v; data[i + 3] = 255
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return Bitmap(width: w, height: h, data: data)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
func testFitMatchesSipsLongEdgeBehaviour() {
|
|
30
|
+
// The real device shapes this has to reproduce.
|
|
31
|
+
XCTAssertEqual(Bitmap.fit(width: 1206, height: 2622, longEdge: 700).0, 322)
|
|
32
|
+
XCTAssertEqual(Bitmap.fit(width: 1206, height: 2622, longEdge: 700).1, 700)
|
|
33
|
+
XCTAssertEqual(Bitmap.fit(width: 2064, height: 2752, longEdge: 700).0, 525)
|
|
34
|
+
XCTAssertEqual(Bitmap.fit(width: 2064, height: 2752, longEdge: 700).1, 700)
|
|
35
|
+
// Smaller than the target is left alone rather than upscaled.
|
|
36
|
+
XCTAssertEqual(Bitmap.fit(width: 100, height: 200, longEdge: 700).1, 200)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
func testGrayGridAveragesToLuminance() {
|
|
40
|
+
XCTAssertEqual(Hashing.grayGrid(solid(20, 20, 255), cols: 2, rows: 2), [255, 255, 255, 255])
|
|
41
|
+
XCTAssertEqual(Hashing.grayGrid(solid(20, 20, 0), cols: 2, rows: 2), [0, 0, 0, 0])
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
func testIdenticalScreensHashIdentically() {
|
|
45
|
+
let a = split(64, 128, left: 20, right: 220)
|
|
46
|
+
let b = split(64, 128, left: 20, right: 220)
|
|
47
|
+
XCTAssertEqual(Hashing.frameHash(a), Hashing.frameHash(b))
|
|
48
|
+
XCTAssertEqual(Hashing.layoutHash(a), Hashing.layoutHash(b))
|
|
49
|
+
XCTAssertEqual(Hashing.signatureDiff(Hashing.regionSignature(a), Hashing.regionSignature(b)), 0)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
func testMirroredLayoutIsFarAway() {
|
|
53
|
+
let a = split(64, 128, left: 20, right: 220)
|
|
54
|
+
let b = split(64, 128, left: 220, right: 20)
|
|
55
|
+
// Well beyond the tolerance of 12 the screen memory uses.
|
|
56
|
+
XCTAssertGreaterThan(Hashing.hashDistance(Hashing.layoutHash(a), Hashing.layoutHash(b)), 20)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
func testHashesAreTheExpectedWidth() {
|
|
60
|
+
let bmp = split(64, 128, left: 20, right: 220)
|
|
61
|
+
XCTAssertEqual(Hashing.frameHash(bmp).count, 32, "128 bits as 32 hex characters")
|
|
62
|
+
XCTAssertEqual(Hashing.layoutHash(bmp).count, 72, "288 bits as 72 hex characters")
|
|
63
|
+
XCTAssertEqual(Hashing.signatureToHex(Hashing.regionSignature(bmp)).count, 64)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
func testSignatureDiffTreatsMissingBaselineAsFullChange() {
|
|
67
|
+
XCTAssertEqual(Hashing.signatureDiff(Hashing.regionSignature(solid(8, 8, 100)), nil), 1)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
func testHashDistanceCountsBits() {
|
|
71
|
+
XCTAssertEqual(Hashing.hashDistance("abcd", "abcd"), 0)
|
|
72
|
+
XCTAssertEqual(Hashing.hashDistance("0", "f"), 4)
|
|
73
|
+
XCTAssertEqual(Hashing.hashDistance("abc", "abcd"), Int.max, "different widths are incomparable")
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
func testStubPlatformProducesReadablePaddedFrames() throws {
|
|
77
|
+
let stub = StubPlatform(width: 120, height: 260)
|
|
78
|
+
_ = try stub.attach(udid: nil)
|
|
79
|
+
let bmp = try stub.withFrame { frame -> Bitmap in
|
|
80
|
+
// The stub pads its rows on purpose: honouring bytesPerRow is the
|
|
81
|
+
// single easiest thing to get wrong against a real surface.
|
|
82
|
+
XCTAssertEqual(frame.bytesPerRow, frame.width * 4 + 40)
|
|
83
|
+
return Bitmap.from(frame, targetLongEdge: 130)
|
|
84
|
+
}
|
|
85
|
+
XCTAssertEqual(bmp.height, 130)
|
|
86
|
+
XCTAssertEqual(bmp.width, 60)
|
|
87
|
+
// Left half dark, right half light — a shear from bad padding would blur this.
|
|
88
|
+
let leftPixel = Int(bmp.data[(65 * bmp.width + 5) * 4])
|
|
89
|
+
let rightPixel = Int(bmp.data[(65 * bmp.width + bmp.width - 5) * 4])
|
|
90
|
+
XCTAssertLessThan(leftPixel, 100)
|
|
91
|
+
XCTAssertGreaterThan(rightPixel, 150)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
func testChangeIsDetectedAcrossStubFrames() throws {
|
|
95
|
+
let stub = StubPlatform(width: 80, height: 160)
|
|
96
|
+
_ = try stub.attach(udid: nil)
|
|
97
|
+
let before = try stub.withFrame { Hashing.regionSignature(Bitmap.from($0, targetLongEdge: 80)) }
|
|
98
|
+
stub.simulateChange()
|
|
99
|
+
let after = try stub.withFrame { Hashing.regionSignature(Bitmap.from($0, targetLongEdge: 80)) }
|
|
100
|
+
XCTAssertGreaterThan(Hashing.signatureDiff(after, before), 0.004)
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/// The Platform surface, exercised through StubPlatform so gestures and app
|
|
105
|
+
/// lifecycle can be tested with no simulator and no private frameworks.
|
|
106
|
+
final class PlatformSurfaceTests: XCTestCase {
|
|
107
|
+
private func stub() throws -> StubPlatform {
|
|
108
|
+
let s = StubPlatform(width: 120, height: 260)
|
|
109
|
+
_ = try s.attach(udid: nil)
|
|
110
|
+
return s
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
func testGesturesReachThePlatform() throws {
|
|
114
|
+
let s = try stub()
|
|
115
|
+
try s.tap(at: CGPoint(x: 10, y: 20), durationMs: 70)
|
|
116
|
+
try s.longPress(at: CGPoint(x: 30, y: 40), durationMs: 600)
|
|
117
|
+
try s.drag(from: CGPoint(x: 1, y: 2), to: CGPoint(x: 3, y: 4), holdMs: 500, durationMs: 400)
|
|
118
|
+
try s.swipe(from: CGPoint(x: 5, y: 6), to: CGPoint(x: 7, y: 8), durationMs: 300)
|
|
119
|
+
XCTAssertEqual(s.recorded, [
|
|
120
|
+
"tap(10,20)",
|
|
121
|
+
"longPress(30,40,600)",
|
|
122
|
+
"drag(1,2->3,4,hold=500)",
|
|
123
|
+
"swipe(5,6->7,8)",
|
|
124
|
+
])
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
func testAppLifecycleReachesThePlatform() throws {
|
|
128
|
+
let s = try stub()
|
|
129
|
+
_ = try s.launch(bundleId: "com.example.app", arguments: ["-a"], environment: ["K": "V"])
|
|
130
|
+
try s.terminate(bundleId: "com.example.app")
|
|
131
|
+
try s.openURL("myapp://x")
|
|
132
|
+
try s.permission(action: "grant", service: "photos", bundleId: "com.example.app")
|
|
133
|
+
XCTAssertEqual(s.recorded, [
|
|
134
|
+
"launch(com.example.app)",
|
|
135
|
+
"terminate(com.example.app)",
|
|
136
|
+
"openURL(myapp://x)",
|
|
137
|
+
"permission(grant,photos,com.example.app)",
|
|
138
|
+
])
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
func testTextRoutesAreDistinct() throws {
|
|
142
|
+
let s = try stub()
|
|
143
|
+
try s.type("abc")
|
|
144
|
+
try s.paste("abc")
|
|
145
|
+
// Two different mechanisms: key events are layout-dependent, paste is not.
|
|
146
|
+
XCTAssertEqual(s.recorded, ["type(abc)", "paste(abc)"])
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
func testAsciiIsTypeableAndEmojiIsNot() {
|
|
150
|
+
XCTAssertTrue(HIDKeyboard.canType("Fryer 3!"))
|
|
151
|
+
XCTAssertFalse(HIDKeyboard.canType("Fryer 🍟"), "emoji has no usage code and must go via paste")
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
func testOnlyVerifiedButtonsHaveCodes() {
|
|
155
|
+
XCTAssertEqual(HIDKeyboard.buttonCode(.home), 2)
|
|
156
|
+
// Unverified codes must refuse rather than guess: a wrong Indigo button
|
|
157
|
+
// can crash backboardd or lock the device.
|
|
158
|
+
for button in [HardwareButton.lock, .siri, .volumeUp, .volumeDown] {
|
|
159
|
+
XCTAssertNil(HIDKeyboard.buttonCode(button), "\(button.rawValue) is not verified")
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/// Motion analysis, on synthetic frames so the expected answer is known.
|
|
165
|
+
final class MotionTests: XCTestCase {
|
|
166
|
+
private func grid(_ fill: (Int, Int) -> UInt8) -> [UInt8] {
|
|
167
|
+
var g = [UInt8](repeating: 0, count: Motion.cols * Motion.rows)
|
|
168
|
+
for r in 0..<Motion.rows { for c in 0..<Motion.cols { g[r * Motion.cols + c] = fill(c, r) } }
|
|
169
|
+
return g
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/// Content that varies in both directions, so a shift has a unique best
|
|
173
|
+
/// match. Horizontally uniform content makes every horizontal offset score
|
|
174
|
+
/// identically, and the search then reports an arbitrary one.
|
|
175
|
+
private func striped(offsetRows: Int = 0) -> [UInt8] {
|
|
176
|
+
grid { c, r in UInt8((((r + offsetRows) * 13) + c * 7) % 256) }
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
func testIdenticalFramesAreStill() {
|
|
180
|
+
XCTAssertEqual(Motion.difference(striped(), striped()), 0)
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
func testSettleNeedsConsecutiveStillFrames() {
|
|
184
|
+
let still = striped()
|
|
185
|
+
var history: [Motion.Frame] = []
|
|
186
|
+
for i in 0..<5 { history.append(Motion.Frame(at: Double(i) * 100, gray: still)) }
|
|
187
|
+
let state = Motion.state(history: history, now: 500)
|
|
188
|
+
XCTAssertTrue(state.settled)
|
|
189
|
+
XCTAssertGreaterThanOrEqual(state.stillFrames, Motion.stillFramesRequired)
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
func testAChangingScreenIsNotSettled() {
|
|
193
|
+
var history: [Motion.Frame] = []
|
|
194
|
+
for i in 0..<5 { history.append(Motion.Frame(at: Double(i) * 100, gray: striped(offsetRows: i * 6))) }
|
|
195
|
+
XCTAssertFalse(Motion.state(history: history, now: 500).settled)
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
func testSpinnerIsNeitherStillNorASceneChange() {
|
|
199
|
+
// Everything holds still except a small patch that keeps changing.
|
|
200
|
+
var history: [Motion.Frame] = []
|
|
201
|
+
for i in 0..<5 {
|
|
202
|
+
var g = striped()
|
|
203
|
+
for r in 40..<45 { for c in 20..<25 { g[r * Motion.cols + c] = UInt8((i * 90) % 256) } }
|
|
204
|
+
history.append(Motion.Frame(at: Double(i) * 100, gray: g))
|
|
205
|
+
}
|
|
206
|
+
let state = Motion.state(history: history, now: 500)
|
|
207
|
+
XCTAssertFalse(state.settled, "a spinner must not read as settled")
|
|
208
|
+
XCTAssertNotNil(state.animatingRegion, "the moving patch should be localised")
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
func testVerticalShiftIsMeasured() {
|
|
212
|
+
let (dx, dy, score) = Motion.shift(striped(), striped(offsetRows: 8))
|
|
213
|
+
XCTAssertEqual(dy, -8, "content moved up by eight rows")
|
|
214
|
+
XCTAssertEqual(dx, 0)
|
|
215
|
+
XCTAssertLessThan(score, 0.02)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
func testScrollIsClassifiedWithAnOffset() throws {
|
|
219
|
+
let t = Motion.classify(before: striped(), after: striped(offsetRows: 8),
|
|
220
|
+
pointHeight: 874, pointWidth: 402)
|
|
221
|
+
XCTAssertEqual(t.kind, .scroll)
|
|
222
|
+
// XCTUnwrap rather than `!`: a force unwrap here kills the whole test
|
|
223
|
+
// process with signal 5 and hides every other result.
|
|
224
|
+
let offset = try XCTUnwrap(t.offset)
|
|
225
|
+
// Eight of ninety-six rows of an 874pt screen.
|
|
226
|
+
XCTAssertEqual(offset.y, -874 * 8 / 96, accuracy: 1)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
func testIdenticalScreensClassifyAsNoChange() {
|
|
230
|
+
let t = Motion.classify(before: striped(), after: striped(), pointHeight: 874, pointWidth: 402)
|
|
231
|
+
XCTAssertEqual(t.kind, .none)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
func testDimmingTheWholeScreenReadsAsSomethingPresenting() {
|
|
235
|
+
let base = striped()
|
|
236
|
+
let dimmed = base.map { UInt8(max(0, Int($0) - 60)) }
|
|
237
|
+
let t = Motion.classify(before: base, after: dimmed, pointHeight: 874, pointWidth: 402)
|
|
238
|
+
XCTAssertTrue([.sheetPresent, .alertPresent].contains(t.kind), "got \(t.kind)")
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/// An accessibility node becoming an element. The subrole matters: a search
|
|
243
|
+
/// field is a TextField to the tree and a SearchField to anyone reading the
|
|
244
|
+
/// map, and losing that costs the reader the only word that identifies it.
|
|
245
|
+
final class AccessibilityElementTests: XCTestCase {
|
|
246
|
+
func testSubroleWinsWhenThereIsOne() {
|
|
247
|
+
let node = AXNode(role: "TextField", subrole: "SearchField", label: nil, value: "Search",
|
|
248
|
+
identifier: "search", enabled: true, selected: false, focused: nil,
|
|
249
|
+
frame: CGRect(x: 32, y: 803, width: 336, height: 38), depth: 2)
|
|
250
|
+
let element = Element(id: 0, node: node)
|
|
251
|
+
XCTAssertEqual(element.role, "SearchField")
|
|
252
|
+
XCTAssertEqual(element.value, "Search")
|
|
253
|
+
XCTAssertEqual(element.identifier, "search")
|
|
254
|
+
XCTAssertEqual(element.state.enabled, true)
|
|
255
|
+
XCTAssertEqual(element.state.selected, false)
|
|
256
|
+
XCTAssertNil(element.state.focused, "unknown stays unknown rather than becoming false")
|
|
257
|
+
XCTAssertEqual(element.source.names, ["ax"])
|
|
258
|
+
XCTAssertEqual(element.center, CGPoint(x: 200, y: 822))
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
func testPlainRoleSurvivesWithNoSubrole() {
|
|
262
|
+
let node = AXNode(role: "Button", label: "Continue",
|
|
263
|
+
frame: CGRect(x: 38, y: 784, width: 326, height: 52), depth: 1)
|
|
264
|
+
let element = Element(id: 3, node: node)
|
|
265
|
+
XCTAssertEqual(element.role, "Button")
|
|
266
|
+
XCTAssertEqual(element.label, "Continue")
|
|
267
|
+
XCTAssertNil(element.identifier)
|
|
268
|
+
XCTAssertNil(element.json["identifier"], "an absent identifier is absent, not null")
|
|
269
|
+
}
|
|
270
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "simframe",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0-rc.1",
|
|
4
4
|
"mcpName": "io.github.lvlrSajjad/simframe",
|
|
5
5
|
"description": "Always-warm iOS Simulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
|
|
6
6
|
"keywords": [
|
|
@@ -40,14 +40,22 @@
|
|
|
40
40
|
},
|
|
41
41
|
"files": [
|
|
42
42
|
"src",
|
|
43
|
-
"native",
|
|
43
|
+
"native/ocr.swift",
|
|
44
|
+
"native/simframed/Package.swift",
|
|
45
|
+
"native/simframed/Sources",
|
|
46
|
+
"native/simframed/Tests",
|
|
47
|
+
"skills",
|
|
48
|
+
"scripts",
|
|
44
49
|
"README.md",
|
|
45
50
|
"LICENSE"
|
|
46
51
|
],
|
|
47
52
|
"scripts": {
|
|
48
|
-
"test": "node --test test/unit.test.mjs",
|
|
53
|
+
"test": "SIMFRAME_HOME=\"$(mktemp -d)\" node --test test/unit.test.mjs",
|
|
49
54
|
"mcp": "node src/cli.js mcp",
|
|
50
|
-
"smoke": "node scripts/smoke.mjs"
|
|
55
|
+
"smoke": "node scripts/smoke.mjs",
|
|
56
|
+
"bench": "./scripts/bench.sh",
|
|
57
|
+
"check:package": "node scripts/check-package.mjs",
|
|
58
|
+
"version": "node scripts/sync-server-version.mjs"
|
|
51
59
|
},
|
|
52
60
|
"dependencies": {
|
|
53
61
|
"@modelcontextprotocol/sdk": "^1.0.0"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Times a realistic navigation flow end to end, and reports where the time went.
|
|
2
|
+
// Used to check that the input driver works at all, and to measure sim_do.
|
|
3
|
+
import { runScript } from '../src/actions.js';
|
|
4
|
+
import * as input from '../src/input.js';
|
|
5
|
+
import * as api from '../src/index.js';
|
|
6
|
+
import { launchApp, terminateApp } from '../src/simctl.js';
|
|
7
|
+
|
|
8
|
+
const BUNDLE = process.argv[2];
|
|
9
|
+
const device = process.argv[3];
|
|
10
|
+
|
|
11
|
+
if (!BUNDLE) {
|
|
12
|
+
console.error('usage: node scripts/bench-flow.mjs <bundle-id> [device]');
|
|
13
|
+
process.exit(2);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const driver = await input.detectDriver();
|
|
17
|
+
console.log(`driver: ${driver.available ? driver.version : driver.reason}`);
|
|
18
|
+
if (!driver.available) process.exit(1);
|
|
19
|
+
|
|
20
|
+
const { device: dev } = await api.ensureDaemon(device);
|
|
21
|
+
console.log(`device: ${dev.name} (${dev.runtime})`);
|
|
22
|
+
const geo = await input.screenInfo(dev.udid);
|
|
23
|
+
console.log(`screen: ${geo.pixelWidth}x${geo.pixelHeight}px @${geo.density}x = ${geo.pointWidth}x${geo.pointHeight}pt\n`);
|
|
24
|
+
|
|
25
|
+
const t0 = Date.now();
|
|
26
|
+
try { await terminateApp(dev.udid, BUNDLE); } catch { /* not running */ }
|
|
27
|
+
await launchApp(dev.udid, BUNDLE);
|
|
28
|
+
const launched = await api.waitFor(device, { mode: 'settle', stableMs: 800, timeoutMs: 20_000 });
|
|
29
|
+
console.log(`cold launch + settle: ${Date.now() - t0}ms (satisfied=${launched.satisfied})`);
|
|
30
|
+
|
|
31
|
+
const nodes = await input.describeAll(dev.udid);
|
|
32
|
+
console.log(`accessibility tree: ${nodes.length} elements`);
|
|
33
|
+
const labelled = nodes.filter((n) => n.label && n.frame.width > 8 && n.frame.height > 8);
|
|
34
|
+
console.log('\nlabelled elements (type, tap point, label):');
|
|
35
|
+
for (const n of labelled.slice(0, 40)) {
|
|
36
|
+
const c = input.centerOf(n);
|
|
37
|
+
console.log(` ${(n.type || '?').padEnd(14)} ${String(`${c.x},${c.y}`).padEnd(10)} ${n.label}`);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Anything sitting in the bottom eighth of the screen is probably a tab bar.
|
|
41
|
+
const tabs = labelled.filter((n) => n.frame.y > geo.pointHeight * 0.86);
|
|
42
|
+
console.log(`\nlikely tab bar: ${tabs.map((t) => t.label).join(', ') || '(none found)'}`);
|
|
43
|
+
|
|
44
|
+
if (tabs.length >= 2) {
|
|
45
|
+
const steps = tabs.slice(0, 4).map((t) => ({ tap: t.label }));
|
|
46
|
+
console.log(`\nrunning ${steps.length} tab switches as ONE sim_do call...`);
|
|
47
|
+
const t1 = Date.now();
|
|
48
|
+
const res = await runScript(device, { steps, stableMs: 500, timeoutMs: 10_000 });
|
|
49
|
+
for (const r of res.results) {
|
|
50
|
+
console.log(` ${r.ok ? 'ok ' : 'FAIL'} ${r.action}: ${r.ok ? r.detail : r.error}` +
|
|
51
|
+
(r.settled ? ` (settled ${r.settled.waitedMs}ms, changed=${r.settled.sawChange})` : ''));
|
|
52
|
+
}
|
|
53
|
+
console.log(`\n${res.ok ? 'flow completed' : 'FLOW FAILED'} — ${res.ranSteps}/${res.totalSteps} steps in ${Date.now() - t1}ms`);
|
|
54
|
+
}
|
package/scripts/bench.sh
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Reproduces every number in docs/BENCHMARKS.md.
|
|
3
|
+
#
|
|
4
|
+
# Needs a booted simulator. Numbers are medians; the first sample of anything
|
|
5
|
+
# involving Vision is discarded, because its warm-up dwarfs the measurement.
|
|
6
|
+
set -uo pipefail
|
|
7
|
+
cd "$(dirname "$0")/.."
|
|
8
|
+
|
|
9
|
+
DEVICE="${1:-}"
|
|
10
|
+
BIN=native/simframed/.build/release/simframed
|
|
11
|
+
|
|
12
|
+
say() { printf '\n\033[1m%s\033[0m\n' "$*"; }
|
|
13
|
+
have() { command -v "$1" >/dev/null 2>&1; }
|
|
14
|
+
|
|
15
|
+
say "environment"
|
|
16
|
+
printf ' chip %s\n' "$(sysctl -n machdep.cpu.brand_string 2>/dev/null)"
|
|
17
|
+
printf ' macOS %s\n' "$(sw_vers -productVersion)"
|
|
18
|
+
printf ' Xcode %s\n' "$(xcodebuild -version 2>/dev/null | head -1)"
|
|
19
|
+
|
|
20
|
+
say "building simframed (release)"
|
|
21
|
+
swift build -c release --package-path native/simframed 2>&1 | tail -1
|
|
22
|
+
|
|
23
|
+
UDID="$DEVICE"
|
|
24
|
+
if [ -z "$UDID" ]; then
|
|
25
|
+
UDID=$($BIN devices | head -1 | awk '{print $1}')
|
|
26
|
+
fi
|
|
27
|
+
if [ -z "$UDID" ]; then
|
|
28
|
+
echo " no booted simulator; boot one and re-run" >&2
|
|
29
|
+
exit 1
|
|
30
|
+
fi
|
|
31
|
+
printf ' device %s\n' "$($BIN devices | grep "$UDID" | cut -c40-)"
|
|
32
|
+
|
|
33
|
+
say "capture pipeline (Phase 0)"
|
|
34
|
+
$BIN bench --udid="$UDID" --n=300 | sed 's/^/ /'
|
|
35
|
+
$BIN bench --udid="$UDID" --n=300 --max-dim=420 | sed 's/^/ /'
|
|
36
|
+
|
|
37
|
+
say "simctl + sips, the path it replaces"
|
|
38
|
+
for _ in 1 2 3 4 5; do
|
|
39
|
+
/usr/bin/time -p sh -c "xcrun simctl io $UDID screenshot --type=png --mask=ignored /tmp/bench.png >/dev/null 2>&1 && sips -Z 700 /tmp/bench.png --out /tmp/bench-s.png >/dev/null 2>&1" 2>&1 | grep real
|
|
40
|
+
done | awk '{s+=$2; n++} END {printf " mean %.0f ms over %d runs\n", s/n*1000, n}'
|
|
41
|
+
|
|
42
|
+
say "hash compatibility (Swift vs JavaScript, needs a still screen)"
|
|
43
|
+
node -e '
|
|
44
|
+
const { execSync } = require("child_process");
|
|
45
|
+
(async () => {
|
|
46
|
+
const api = await import("./src/index.js");
|
|
47
|
+
const { hashDistance } = await import("./src/analyze.js");
|
|
48
|
+
let a, b;
|
|
49
|
+
for (let i = 0; i < 25; i++) {
|
|
50
|
+
a = (await api.getState(undefined, {})).state;
|
|
51
|
+
await new Promise(r => setTimeout(r, 1000));
|
|
52
|
+
b = (await api.getState(undefined, {})).state;
|
|
53
|
+
if (a.hash === b.hash && a.layoutHash === b.layoutHash) break;
|
|
54
|
+
}
|
|
55
|
+
if (a.hash !== b.hash) { console.log(" screen never settled; skipped"); return; }
|
|
56
|
+
const sw = JSON.parse(execSync("native/simframed/.build/release/simframed hash").toString());
|
|
57
|
+
console.log(` frameHash delta ${hashDistance(b.hash, sw.hash)}/128`);
|
|
58
|
+
console.log(` layoutHash delta ${hashDistance(b.layoutHash, sw.layoutHash)}/288`);
|
|
59
|
+
})()' 2>/dev/null
|
|
60
|
+
|
|
61
|
+
say "control socket round trips (Phase 1)"
|
|
62
|
+
node -e '
|
|
63
|
+
import("./src/control.js").then(async (c) => {
|
|
64
|
+
const UD = process.argv[1];
|
|
65
|
+
const med = a => { const s=[...a].sort((x,y)=>x-y); return s[s.length>>1]; };
|
|
66
|
+
const bench = async (label, fn, n) => {
|
|
67
|
+
const t = []; for (let i=0;i<n;i++){ const t0=Date.now(); await fn(i); t.push(Date.now()-t0); }
|
|
68
|
+
console.log(` ${label.padEnd(22)} n=${String(n).padStart(3)} median ${med(t)}ms`);
|
|
69
|
+
};
|
|
70
|
+
await bench("ping", () => c.request(UD,{action:"ping"}), 100);
|
|
71
|
+
await bench("status", () => c.status(UD), 50);
|
|
72
|
+
await bench("tap (70ms hold)", i => c.tap(UD, 200+(i%3), 300), 100);
|
|
73
|
+
await bench("tap (10ms hold)", i => c.tap(UD, 200+(i%3), 300, {durationMs:10}), 100);
|
|
74
|
+
})' "$UDID" 2>/dev/null
|
|
75
|
+
|
|
76
|
+
say "text recognition (Phase 2)"
|
|
77
|
+
node -e '
|
|
78
|
+
import("./src/control.js").then(async (c) => {
|
|
79
|
+
const UD = process.argv[1];
|
|
80
|
+
const med = a => { const s=[...a].sort((x,y)=>x-y); return s[s.length>>1]; };
|
|
81
|
+
const t=[]; for (let i=0;i<12;i++){ const t0=Date.now(); await c.request(UD,{action:"ui"}); t.push(Date.now()-t0); }
|
|
82
|
+
// Drop the first: Vision warm-up dwarfs it.
|
|
83
|
+
console.log(` in-process first ${t[0]}ms median ${med(t.slice(1))}ms`);
|
|
84
|
+
})' "$UDID" 2>/dev/null
|
|
85
|
+
|
|
86
|
+
say "screen map build (Phase 2)"
|
|
87
|
+
node -e '
|
|
88
|
+
import("./src/index.js").then(async (api) => {
|
|
89
|
+
const UD = process.argv[1];
|
|
90
|
+
for (const pass of [1,2,3]) {
|
|
91
|
+
api.screenmap.forget(UD);
|
|
92
|
+
const t0 = Date.now();
|
|
93
|
+
try { await api.locate(undefined, "a", {}); } catch { /* the label need not exist */ }
|
|
94
|
+
console.log(` build ${pass}: ${Date.now()-t0}ms`);
|
|
95
|
+
}
|
|
96
|
+
})' "$UDID" 2>/dev/null
|
|
97
|
+
|
|
98
|
+
say "done — compare against docs/BENCHMARKS.md"
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Does the tarball contain everything the installed tool needs?
|
|
3
|
+
//
|
|
4
|
+
// This is the cheapest catch for the two worst regressions this project has
|
|
5
|
+
// shipped, both of which passed every test and printed nothing:
|
|
6
|
+
//
|
|
7
|
+
// - `Tests/` was absent, so Package.swift declared a test target with no
|
|
8
|
+
// directory, SwiftPM reported overlapping sources, and every install
|
|
9
|
+
// silently fell back to the simctl engine.
|
|
10
|
+
// - `native/ocr.swift` was absent, so OCR was silently off for all users.
|
|
11
|
+
//
|
|
12
|
+
// The required list is DERIVED from what the build actually reads, never
|
|
13
|
+
// hand-maintained. A hand-written list is how `files` rotted in the first
|
|
14
|
+
// place: it was correct when written and wrong three commits later. Add a new
|
|
15
|
+
// Swift source or a new module here and it becomes required automatically.
|
|
16
|
+
import { execFileSync } from 'node:child_process';
|
|
17
|
+
import fs from 'node:fs';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import { fileURLToPath } from 'node:url';
|
|
20
|
+
|
|
21
|
+
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
22
|
+
const rel = (p) => path.relative(ROOT, p).split(path.sep).join('/');
|
|
23
|
+
|
|
24
|
+
function walk(dir, filter, out = []) {
|
|
25
|
+
let entries;
|
|
26
|
+
try {
|
|
27
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
28
|
+
} catch {
|
|
29
|
+
return out;
|
|
30
|
+
}
|
|
31
|
+
for (const e of entries) {
|
|
32
|
+
const full = path.join(dir, e.name);
|
|
33
|
+
// .build holds compiled output, not inputs, and is never published.
|
|
34
|
+
if (e.isDirectory()) {
|
|
35
|
+
if (e.name === '.build' || e.name === 'node_modules') continue;
|
|
36
|
+
walk(full, filter, out);
|
|
37
|
+
} else if (filter(full)) {
|
|
38
|
+
out.push(full);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return out;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Everything the SwiftPM build and the Node runtime read at install or run time. */
|
|
45
|
+
function requiredFiles() {
|
|
46
|
+
const swiftPackage = path.join(ROOT, 'native', 'simframed');
|
|
47
|
+
const required = new Set();
|
|
48
|
+
|
|
49
|
+
// The SwiftPM manifest, and every source under the targets it declares.
|
|
50
|
+
required.add(rel(path.join(swiftPackage, 'Package.swift')));
|
|
51
|
+
for (const dir of ['Sources', 'Tests']) {
|
|
52
|
+
for (const f of walk(path.join(swiftPackage, dir), (p) => p.endsWith('.swift'))) {
|
|
53
|
+
required.add(rel(f));
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
// Test targets are build inputs whether or not anyone runs them: a declared
|
|
57
|
+
// target with a missing directory fails the whole build.
|
|
58
|
+
if (!walk(path.join(swiftPackage, 'Tests'), (p) => p.endsWith('.swift')).length) {
|
|
59
|
+
throw new Error('no Swift test sources found — has the layout moved? This check would silently pass.');
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// The standalone OCR helper, compiled on demand by src/ocr.js.
|
|
63
|
+
required.add(rel(path.join(ROOT, 'native', 'ocr.swift')));
|
|
64
|
+
|
|
65
|
+
// Every runtime module. `files` ships "src" wholesale today; asserting each
|
|
66
|
+
// one catches anybody narrowing that later.
|
|
67
|
+
for (const f of walk(path.join(ROOT, 'src'), (p) => p.endsWith('.js'))) required.add(rel(f));
|
|
68
|
+
|
|
69
|
+
// The Claude Code skill. It is the low-token path this tool is designed
|
|
70
|
+
// around, and an installed copy without it is an installation of half the
|
|
71
|
+
// idea — silently, which is the failure mode this whole check exists for.
|
|
72
|
+
for (const f of walk(path.join(ROOT, 'skills'), (p) => p.endsWith('.md'))) required.add(rel(f));
|
|
73
|
+
if (!walk(path.join(ROOT, 'skills'), (p) => p.endsWith('.md')).length) {
|
|
74
|
+
throw new Error('no skill found under skills/ — has the layout moved? This check would silently pass.');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Anything package.json points at to run.
|
|
78
|
+
const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
|
|
79
|
+
for (const target of Object.values(pkg.bin ?? {})) required.add(target.replace(/^\.\//, ''));
|
|
80
|
+
if (pkg.main) required.add(pkg.main.replace(/^\.\//, ''));
|
|
81
|
+
|
|
82
|
+
return [...required].sort();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const required = requiredFiles();
|
|
86
|
+
const packed = JSON.parse(execFileSync('npm', ['pack', '--dry-run', '--json'], { cwd: ROOT, encoding: 'utf8' }));
|
|
87
|
+
const shipped = new Set(packed[0].files.map((f) => f.path));
|
|
88
|
+
const missing = required.filter((f) => !shipped.has(f));
|
|
89
|
+
|
|
90
|
+
console.log(`${required.length} build inputs required, ${shipped.size} files in the tarball`);
|
|
91
|
+
for (const f of missing) console.log(`MISSING ${f}`);
|
|
92
|
+
if (missing.length) {
|
|
93
|
+
console.error(
|
|
94
|
+
`\n${missing.length} file(s) the build needs are absent from the published package.` +
|
|
95
|
+
`\nThis does not fail a build or a test — it fails silently for whoever installs it.`,
|
|
96
|
+
);
|
|
97
|
+
process.exit(1);
|
|
98
|
+
}
|
|
99
|
+
console.log('ok — every build input ships');
|