simframe 0.7.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -4
- package/flows/hpi-suite.json +68 -0
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +49 -1
- package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +7 -0
- package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +11 -0
- package/native/simframed/Sources/SimframeCore/CaptureRecovery.swift +48 -0
- package/native/simframed/Sources/simframed/main.swift +128 -73
- package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +40 -0
- package/package.json +2 -1
- package/scripts/bench-hpi.mjs +254 -0
- package/scripts/check-package.mjs +7 -0
- package/scripts/ci-memory.mjs +15 -2
- package/src/actions.js +184 -4
- package/src/baseline.js +333 -0
- package/src/cli.js +336 -2
- package/src/daemon.js +9 -0
- package/src/fingerprint.js +7 -1
- package/src/graph.js +162 -1
- package/src/index.js +118 -20
- package/src/input.js +111 -1
- package/src/intent.js +11 -2
- package/src/matching.js +81 -2
- package/src/mcp.js +14 -1
- package/src/metrics.js +499 -0
- package/src/navigate.js +44 -7
- package/src/platform/android.js +27 -0
- package/src/platform/index.js +3 -0
- package/src/platform/ios.js +63 -0
- package/src/screenmap.js +36 -14
- package/src/view.js +4 -3
package/README.md
CHANGED
|
@@ -37,6 +37,7 @@ Same four-tab navigation flow, on a real production app:
|
|
|
37
37
|
| | Before | With simframe |
|
|
38
38
|
| --- | --- | --- |
|
|
39
39
|
| Look at the screen | ~130–400 ms, blocking | **~20 ms**, already captured |
|
|
40
|
+
|
|
40
41
|
| "Did anything change?" | a full image | **~2 ms**, text only |
|
|
41
42
|
| Finding a control | read tree (~570 ms) + reason | **~1 ms** from memory |
|
|
42
43
|
| A 4-step flow, verified | 4+ model round trips | **1 call**, 3.6 s |
|
|
@@ -44,6 +45,13 @@ Same four-tab navigation flow, on a real production app:
|
|
|
44
45
|
| A 10-step flow | 10 turns, 10 images (~16,000 tokens at best) | **1 turn, 0 images, ~1,650 characters** |
|
|
45
46
|
| Reading a screen | an image: ~1,600 tokens, no tap points | **~330 tokens** of text, with tap points |
|
|
46
47
|
|
|
48
|
+
Every figure above is the cost inside a live process — the MCP server, or the
|
|
49
|
+
daemon answering a socket — which is how an agent actually uses simframe. A
|
|
50
|
+
one-shot `simframe` command from a shell pays about 200 ms of Node startup on
|
|
51
|
+
top, and a frame sitting on an idle screen can be older than 20 ms because the
|
|
52
|
+
capture loop throttles when nothing moves. `~20 ms` is the read, not the
|
|
53
|
+
process.
|
|
54
|
+
|
|
47
55
|
The four-tab tour, three times back to back from a cleared memory:
|
|
48
56
|
|
|
49
57
|
| Pass | Wall clock | Steps verified | Controls from memory |
|
|
@@ -67,10 +75,16 @@ npm install -g simframe
|
|
|
67
75
|
simframe doctor
|
|
68
76
|
```
|
|
69
77
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
78
|
+
**Whichever of those two commands you run first** builds a small Swift daemon
|
|
79
|
+
from source — including `doctor`, which is why a cold `doctor` takes around 15
|
|
80
|
+
seconds and every later one takes two. It needs the Xcode command line tools,
|
|
81
|
+
which you already have if you have a simulator. Without them simframe falls
|
|
82
|
+
back to the original `simctl` loop and says so.
|
|
83
|
+
|
|
84
|
+
If more than one simulator is booted, name the one you mean — `--device=<udid>`,
|
|
85
|
+
or `export SIMFRAME_DEVICE=<udid>` once per shell. simframe refuses to choose
|
|
86
|
+
for you, because the first booted device is nobody's idea of "yours" and the
|
|
87
|
+
command that would act on it is a tap.
|
|
74
88
|
|
|
75
89
|
`doctor` checks each capability separately and tells you what you have:
|
|
76
90
|
|
|
@@ -527,6 +541,9 @@ simframe recall --ago=15000 # the frame from 15s ago
|
|
|
527
541
|
simframe frame --out=now.png # newest frame, native resolution, to a file
|
|
528
542
|
simframe strip --count=6 # contact sheet, for an animation
|
|
529
543
|
simframe doctor --strict # any degraded layer is a non-zero exit
|
|
544
|
+
simframe escalations # why simframe still needs a model, by reason
|
|
545
|
+
simframe hpi # speed and accuracy against a human baseline
|
|
546
|
+
simframe baseline record settings-larger-text --runs=5 # record the human
|
|
530
547
|
simframe start / status / stop [--force] / devices
|
|
531
548
|
simframe ui --device=emulator-5554 # or export SIMFRAME_DEVICE once
|
|
532
549
|
```
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"name": "settings-larger-text",
|
|
4
|
+
"note": "Four steps into a nested Settings list. The two Settings sublists are the fingerprint pair that needs a nav title to tell them apart, so this flow exercises screen identity as well as navigation.",
|
|
5
|
+
"minSteps": 4,
|
|
6
|
+
"endsOn": "Larger Text",
|
|
7
|
+
"human": [
|
|
8
|
+
"Tap Settings on the home screen.",
|
|
9
|
+
"Tap Accessibility.",
|
|
10
|
+
"Tap Display & Text Size.",
|
|
11
|
+
"Tap Larger Text, and stop there."
|
|
12
|
+
],
|
|
13
|
+
"reset": {
|
|
14
|
+
"terminate": [
|
|
15
|
+
"com.apple.Preferences"
|
|
16
|
+
],
|
|
17
|
+
"home": true,
|
|
18
|
+
"launch": "com.apple.Preferences",
|
|
19
|
+
"rootMarker": "Accessibility",
|
|
20
|
+
"maxBack": 4
|
|
21
|
+
},
|
|
22
|
+
"steps": [
|
|
23
|
+
{
|
|
24
|
+
"launch": {
|
|
25
|
+
"value": "com.apple.Preferences"
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"tap": "Accessibility"
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"tap": "Display & Text Size"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"tap": "Larger Text"
|
|
36
|
+
}
|
|
37
|
+
]
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"name": "contacts-kate-bell",
|
|
41
|
+
"note": "An indexed list with a search field, two steps deep. Short on purpose: a flow this shallow is where a per-step overhead shows up as a ratio rather than being absorbed by navigation.",
|
|
42
|
+
"minSteps": 2,
|
|
43
|
+
"endsOn": "Kate Bell",
|
|
44
|
+
"human": [
|
|
45
|
+
"Tap Contacts on the home screen.",
|
|
46
|
+
"Tap Kate Bell, and stop there."
|
|
47
|
+
],
|
|
48
|
+
"reset": {
|
|
49
|
+
"terminate": [
|
|
50
|
+
"com.apple.MobileAddressBook"
|
|
51
|
+
],
|
|
52
|
+
"home": true,
|
|
53
|
+
"launch": "com.apple.MobileAddressBook",
|
|
54
|
+
"rootMarker": "Kate Bell",
|
|
55
|
+
"maxBack": 3
|
|
56
|
+
},
|
|
57
|
+
"steps": [
|
|
58
|
+
{
|
|
59
|
+
"launch": {
|
|
60
|
+
"value": "com.apple.MobileAddressBook"
|
|
61
|
+
}
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"tap": "Kate Bell"
|
|
65
|
+
}
|
|
66
|
+
]
|
|
67
|
+
}
|
|
68
|
+
]
|
|
@@ -157,6 +157,21 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
|
|
|
157
157
|
return try resolveDisplay(on: device, warmInput: true)
|
|
158
158
|
}
|
|
159
159
|
|
|
160
|
+
/// Rebind to the device from scratch: a fresh device object, a fresh port.
|
|
161
|
+
///
|
|
162
|
+
/// `reattachDisplay` reuses the cached `device`, which is right for a port
|
|
163
|
+
/// that was rebuilt under a living device and wrong for everything else —
|
|
164
|
+
/// the device object itself can be stale after a restart, and re-walking
|
|
165
|
+
/// its `ioPorts` then re-finds the same dead descriptors. This asks
|
|
166
|
+
/// CoreSimulator for the device list again, so nothing from the previous
|
|
167
|
+
/// session survives. Input is warmed too, because a session that outlived
|
|
168
|
+
/// its device is dead anyway.
|
|
169
|
+
public func reattachDevice(udid: String?) throws -> DeviceInfo {
|
|
170
|
+
device = nil
|
|
171
|
+
display = nil
|
|
172
|
+
return try attach(udid: udid)
|
|
173
|
+
}
|
|
174
|
+
|
|
160
175
|
/// Re-resolve the display port on the device we are already bound to.
|
|
161
176
|
///
|
|
162
177
|
/// Input is deliberately left alone: the HID session is independent of the
|
|
@@ -180,15 +195,36 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
|
|
|
180
195
|
let sizeSel = NSSelectorFromString("displaySize")
|
|
181
196
|
typealias SizeFn = @convention(c) (AnyObject, Selector) -> CGSize
|
|
182
197
|
|
|
198
|
+
// Two passes, because a size is a claim and a surface is evidence.
|
|
199
|
+
//
|
|
200
|
+
// A torn-down port keeps reporting a real `displaySize` while
|
|
201
|
+
// `framebufferSurface` returns nil for the rest of the device's life.
|
|
202
|
+
// Selecting on size alone therefore reattached to the dead port and
|
|
203
|
+
// declared success: the daemon logged "re-resolved the display port
|
|
204
|
+
// after 6 failed reads" and then failed six more, in a loop, until the
|
|
205
|
+
// device was restarted. That is the pathology CaptureRecovery's
|
|
206
|
+
// `stalledAfterReattaches` exists to *notice*; this is what fixes it.
|
|
207
|
+
//
|
|
208
|
+
// The size check stays as the cheap pre-filter. The second pass is the
|
|
209
|
+
// one that decides, and if no candidate yields a surface the first
|
|
210
|
+
// plausible one is used anyway — during boot the port is real and the
|
|
211
|
+
// surface has simply not arrived yet, and refusing to attach then
|
|
212
|
+
// would trade a recoverable wedge for a daemon that never starts.
|
|
213
|
+
var candidates: [NSObject] = []
|
|
183
214
|
for port in ports {
|
|
184
215
|
guard port.responds(to: descriptorSel),
|
|
185
216
|
let descriptor = port.perform(descriptorSel)?.takeUnretainedValue() as? NSObject,
|
|
186
217
|
descriptor.conforms(to: proto),
|
|
187
218
|
descriptor.responds(to: sizeSel),
|
|
188
219
|
let sizeImp = descriptor.method(for: sizeSel) else { continue }
|
|
189
|
-
// Several ports conform; only the live one reports a real size.
|
|
190
220
|
let size = unsafeBitCast(sizeImp, to: SizeFn.self)(descriptor, sizeSel)
|
|
191
221
|
guard size.width > 0, size.height > 0 else { continue }
|
|
222
|
+
candidates.append(descriptor)
|
|
223
|
+
}
|
|
224
|
+
let live = candidates.first(where: Self.yieldsSurface) ?? candidates.first
|
|
225
|
+
for descriptor in candidates where descriptor === live {
|
|
226
|
+
let sizeImp = descriptor.method(for: sizeSel)!
|
|
227
|
+
let size = unsafeBitCast(sizeImp, to: SizeFn.self)(descriptor, sizeSel)
|
|
192
228
|
display = descriptor
|
|
193
229
|
// The bridge captures one device's token and installs itself on a
|
|
194
230
|
// process-wide translator, so it belongs to the device it was built
|
|
@@ -259,6 +295,18 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
|
|
|
259
295
|
}
|
|
260
296
|
}
|
|
261
297
|
|
|
298
|
+
/// Does this display descriptor actually produce a framebuffer?
|
|
299
|
+
///
|
|
300
|
+
/// The single question that separates a live port from a torn-down one, and
|
|
301
|
+
/// it is not the one the resolver used to ask. Cheap: one selector call and
|
|
302
|
+
/// a cast, no lock, no pixels read.
|
|
303
|
+
static func yieldsSurface(_ descriptor: NSObject) -> Bool {
|
|
304
|
+
guard descriptor.responds(to: NSSelectorFromString("framebufferSurface")),
|
|
305
|
+
let raw = descriptor.perform(NSSelectorFromString("framebufferSurface"))?.takeUnretainedValue()
|
|
306
|
+
else { return false }
|
|
307
|
+
return raw is IOSurface
|
|
308
|
+
}
|
|
309
|
+
|
|
262
310
|
public func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T {
|
|
263
311
|
guard let display else { throw PrivateAPIError.noDisplayPort }
|
|
264
312
|
guard let raw = display.perform(NSSelectorFromString("framebufferSurface"))?.takeUnretainedValue(),
|
|
@@ -84,6 +84,13 @@ public protocol SimulatorPlatform: AnyObject {
|
|
|
84
84
|
/// perfectly visible, and a daemon restart fixed it instantly. Without a
|
|
85
85
|
/// way to re-resolve, a restart is the only cure.
|
|
86
86
|
func reattachDisplay() throws -> DeviceInfo
|
|
87
|
+
/// Rebind from scratch: a fresh device object as well as a fresh port.
|
|
88
|
+
///
|
|
89
|
+
/// The escalation for when re-resolving the port has demonstrably not
|
|
90
|
+
/// helped. `reattachDisplay` re-walks the cached device's ports, which
|
|
91
|
+
/// re-finds the same dead descriptors when it is the device binding that
|
|
92
|
+
/// is stale.
|
|
93
|
+
func reattachDevice(udid: String?) throws -> DeviceInfo
|
|
87
94
|
/// Borrow the current framebuffer. The pointer is only valid inside `body`.
|
|
88
95
|
func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T
|
|
89
96
|
/// Called whenever the display reports damage — the per-redraw signal, so a
|
|
@@ -33,6 +33,10 @@ public final class StubPlatform: SimulatorPlatform {
|
|
|
33
33
|
/// Counted so a test can assert the capture loop actually tries to recover
|
|
34
34
|
/// rather than logging the same failure forever.
|
|
35
35
|
public private(set) var reattachCount = 0
|
|
36
|
+
/// Full rebinds, counted separately: the escalation is a different act
|
|
37
|
+
/// from re-resolving a port and a test that cannot tell them apart cannot
|
|
38
|
+
/// assert the escalation happened.
|
|
39
|
+
public private(set) var rebindCount = 0
|
|
36
40
|
private var attachedUdid: String?
|
|
37
41
|
|
|
38
42
|
/// Counted, so a test can assert that a dead input path is actually retried.
|
|
@@ -50,6 +54,13 @@ public final class StubPlatform: SimulatorPlatform {
|
|
|
50
54
|
return DeviceInfo(udid: attachedUdid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
|
|
51
55
|
}
|
|
52
56
|
|
|
57
|
+
public func reattachDevice(udid: String?) throws -> DeviceInfo {
|
|
58
|
+
rebindCount += 1
|
|
59
|
+
if failReattach { throw PrivateAPIError.noDisplayPort }
|
|
60
|
+
attachedUdid = udid ?? attachedUdid
|
|
61
|
+
return DeviceInfo(udid: attachedUdid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
|
|
62
|
+
}
|
|
63
|
+
|
|
53
64
|
public func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T {
|
|
54
65
|
let bytesPerRow = width * 4 + 40 // deliberate padding: mirrors real surfaces
|
|
55
66
|
if buffer.count != bytesPerRow * height {
|
|
@@ -36,9 +36,19 @@ public struct CaptureRecovery {
|
|
|
36
36
|
/// failure count does keep growing because nothing resets it.
|
|
37
37
|
public static let stalledAfterFailures = reattachAfterFailures * 3
|
|
38
38
|
|
|
39
|
+
/// How many times to rebind the device per stall episode.
|
|
40
|
+
///
|
|
41
|
+
/// Bounded because a rebind asks CoreSimulator for the whole device list
|
|
42
|
+
/// and warms input: worth doing when re-resolving has failed twice, not
|
|
43
|
+
/// worth doing every half second forever. Two attempts, then the loop goes
|
|
44
|
+
/// back to reporting the state it is in.
|
|
45
|
+
public static let maxRebinds = 2
|
|
46
|
+
|
|
39
47
|
public private(set) var consecutiveFailures = 0
|
|
40
48
|
/// Successful re-resolves since the last real frame.
|
|
41
49
|
public private(set) var reattaches = 0
|
|
50
|
+
/// Full rebinds since the last real frame.
|
|
51
|
+
public private(set) var rebinds = 0
|
|
42
52
|
private let threshold: Int
|
|
43
53
|
|
|
44
54
|
public init(threshold: Int = CaptureRecovery.reattachAfterFailures) {
|
|
@@ -54,12 +64,24 @@ public struct CaptureRecovery {
|
|
|
54
64
|
reattaches >= Self.stalledAfterReattaches || consecutiveFailures >= Self.stalledAfterFailures
|
|
55
65
|
}
|
|
56
66
|
|
|
67
|
+
/// Has re-resolving the port had its chance?
|
|
68
|
+
///
|
|
69
|
+
/// Two successful re-resolves with no frame between them is the port
|
|
70
|
+
/// telling us it was never the problem. That was already the *stalled*
|
|
71
|
+
/// signal; now it is also the trigger to try the one thing that had only
|
|
72
|
+
/// ever been done by hand — rebinding to the device, which is what
|
|
73
|
+
/// restarting the daemon did.
|
|
74
|
+
public var needsRebind: Bool {
|
|
75
|
+
reattaches >= Self.stalledAfterReattaches && rebinds < Self.maxRebinds
|
|
76
|
+
}
|
|
77
|
+
|
|
57
78
|
public mutating func captureSucceeded() {
|
|
58
79
|
consecutiveFailures = 0
|
|
59
80
|
// A real frame is the only evidence that health is back. Resetting this
|
|
60
81
|
// anywhere else — on a re-resolve, say — is how the loop above stayed
|
|
61
82
|
// invisible.
|
|
62
83
|
reattaches = 0
|
|
84
|
+
rebinds = 0
|
|
63
85
|
}
|
|
64
86
|
|
|
65
87
|
/// Records a failure and says whether the port is now due a re-resolve.
|
|
@@ -91,4 +113,30 @@ public struct CaptureRecovery {
|
|
|
91
113
|
return .failure(error)
|
|
92
114
|
}
|
|
93
115
|
}
|
|
116
|
+
|
|
117
|
+
/// Rebind to the device itself, and re-arm the callback on the new port.
|
|
118
|
+
///
|
|
119
|
+
/// The escalation `needsRebind` gates. Re-arming matters here for the same
|
|
120
|
+
/// reason it does in `reattach`: a fresh descriptor with no callback on it
|
|
121
|
+
/// is a daemon that has recovered and will never notice another change,
|
|
122
|
+
/// which looks exactly like the failure it just recovered from.
|
|
123
|
+
public mutating func rebind(
|
|
124
|
+
platform: SimulatorPlatform,
|
|
125
|
+
udid: String?,
|
|
126
|
+
onDamage: @escaping () -> Void
|
|
127
|
+
) -> Result<Int, Error> {
|
|
128
|
+
let failures = consecutiveFailures
|
|
129
|
+
rebinds += 1
|
|
130
|
+
do {
|
|
131
|
+
_ = try platform.reattachDevice(udid: udid)
|
|
132
|
+
try platform.observeChanges(onDamage)
|
|
133
|
+
consecutiveFailures = 0
|
|
134
|
+
// `reattaches` is deliberately left alone. It is the evidence that
|
|
135
|
+
// the port was not the problem, and a rebind does not make that
|
|
136
|
+
// untrue — only a real frame does, in captureSucceeded().
|
|
137
|
+
return .success(failures)
|
|
138
|
+
} catch {
|
|
139
|
+
return .failure(error)
|
|
140
|
+
}
|
|
141
|
+
}
|
|
94
142
|
}
|
|
@@ -3,6 +3,25 @@ import Foundation
|
|
|
3
3
|
import PrivateAPI
|
|
4
4
|
import SimframeCore
|
|
5
5
|
|
|
6
|
+
/// Resident size of this process, in bytes, or 0 if the kernel will not say.
|
|
7
|
+
///
|
|
8
|
+
/// Logged with throughput because the capture wedge has no established cause
|
|
9
|
+
/// and memory pressure is one of two candidates. A number in the log every
|
|
10
|
+
/// second is what lets the next wedge be correlated with a spike — or clear
|
|
11
|
+
/// memory of suspicion, which is just as useful.
|
|
12
|
+
func residentBytes() -> UInt64 {
|
|
13
|
+
var info = mach_task_basic_info()
|
|
14
|
+
var count = mach_msg_type_number_t(
|
|
15
|
+
MemoryLayout<mach_task_basic_info>.size / MemoryLayout<natural_t>.size
|
|
16
|
+
)
|
|
17
|
+
let result = withUnsafeMutablePointer(to: &info) { pointer in
|
|
18
|
+
pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { rebound in
|
|
19
|
+
task_info(mach_task_self_, task_flavor_t(MACH_TASK_BASIC_INFO), rebound, &count)
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return result == KERN_SUCCESS ? info.resident_size : 0
|
|
23
|
+
}
|
|
24
|
+
|
|
6
25
|
let args = Array(CommandLine.arguments.dropFirst())
|
|
7
26
|
func flag(_ name: String) -> String? {
|
|
8
27
|
guard let a = args.first(where: { $0.hasPrefix("--\(name)=") }) else { return nil }
|
|
@@ -405,6 +424,15 @@ case "run":
|
|
|
405
424
|
}
|
|
406
425
|
defer { control.stop() }
|
|
407
426
|
|
|
427
|
+
// A new capture session inherits no stall.
|
|
428
|
+
//
|
|
429
|
+
// capture-health.json is cleared when a stalled loop captures a frame
|
|
430
|
+
// again, and a loop that dies while stalled never gets to. So a fresh
|
|
431
|
+
// daemon captured happily while `doctor` reported "stalled for 221s,
|
|
432
|
+
// 60 re-attaches" from its dead predecessor — and the display probe,
|
|
433
|
+
// correctly on that input, called it a simframe bug. It was: this one.
|
|
434
|
+
try? store.writeCaptureHealth(nil)
|
|
435
|
+
|
|
408
436
|
FileHandle.standardError.write("simframed: capturing \(device.name) (\(device.udid))\n".data(using: .utf8)!)
|
|
409
437
|
|
|
410
438
|
while true {
|
|
@@ -417,80 +445,103 @@ case "run":
|
|
|
417
445
|
if due {
|
|
418
446
|
lock.lock(); dirty = false; lock.unlock()
|
|
419
447
|
let t0 = DispatchTime.now().uptimeNanoseconds
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
// right; never recovering from it is not, so re-resolve the
|
|
459
|
-
// port and re-arm the damage callback.
|
|
460
|
-
if due {
|
|
461
|
-
switch recovery.reattach(platform: platform, onDamage: {
|
|
462
|
-
lock.lock(); dirty = true; lock.unlock()
|
|
463
|
-
}) {
|
|
464
|
-
case .success(let after):
|
|
465
|
-
lock.lock(); dirty = true; lock.unlock()
|
|
466
|
-
FileHandle.standardError.write(
|
|
467
|
-
"simframed: re-resolved the display port after \(after) failed reads\n"
|
|
468
|
-
.data(using: .utf8)!)
|
|
469
|
-
case .failure(let error):
|
|
448
|
+
// One pool per capture.
|
|
449
|
+
//
|
|
450
|
+
// There was none anywhere in this loop, and on Darwin that is
|
|
451
|
+
// the standard way to get the working set this daemon had:
|
|
452
|
+
// 732 MB resident after eleven minutes and 2831 frames, for a
|
|
453
|
+
// process that holds one frame at a time. CoreGraphics scaling
|
|
454
|
+
// and IOSurface access both produce autoreleased temporaries,
|
|
455
|
+
// and a `while true` loop with no pool of its own drains
|
|
456
|
+
// nothing. Whether that pressure is what wedges capture is
|
|
457
|
+
// unproven — the RSS also *fell* 200 MB in 25 s, so nothing is
|
|
458
|
+
// leaking monotonically — but a frame grabber should not hold
|
|
459
|
+
// half a gigabyte either way.
|
|
460
|
+
autoreleasepool {
|
|
461
|
+
do {
|
|
462
|
+
// One grab produces both sizes; the surface pointer is only
|
|
463
|
+
// valid inside this call, so nothing may be deferred out of it.
|
|
464
|
+
let wantFull = store.wantsFullFrame()
|
|
465
|
+
let (bmp, full) = try platform.withFrame { frame -> (Bitmap, Bitmap?) in
|
|
466
|
+
let scaled = CoreGraphicsScaler.bitmap(from: frame, targetLongEdge: longEdge)
|
|
467
|
+
?? Bitmap.from(frame, targetLongEdge: longEdge)
|
|
468
|
+
let native = wantFull
|
|
469
|
+
? CoreGraphicsScaler.bitmap(from: frame, targetLongEdge: max(frame.width, frame.height))
|
|
470
|
+
: nil
|
|
471
|
+
return (scaled, native)
|
|
472
|
+
}
|
|
473
|
+
let ms = Double(DispatchTime.now().uptimeNanoseconds - t0) / 1e6
|
|
474
|
+
try store.record(bmp, fullBitmap: full, captureMs: ms)
|
|
475
|
+
latencies.append(Double(DispatchTime.now().uptimeNanoseconds - t0) / 1e6)
|
|
476
|
+
frames += 1
|
|
477
|
+
lastCapture = now
|
|
478
|
+
let wasStalled = recovery.isStalled
|
|
479
|
+
recovery.captureSucceeded()
|
|
480
|
+
if wasStalled {
|
|
481
|
+
// A frame after a stall is the only thing that clears
|
|
482
|
+
// it, and it is worth saying out loud: the device came
|
|
483
|
+
// back on its own, which nobody would otherwise know.
|
|
484
|
+
try? store.writeCaptureHealth(nil)
|
|
485
|
+
stalledSince = nil
|
|
470
486
|
FileHandle.standardError.write(
|
|
471
|
-
"simframed:
|
|
487
|
+
"simframed: capture recovered on its own\n".data(using: .utf8)!)
|
|
472
488
|
}
|
|
489
|
+
} catch {
|
|
490
|
+
let due = recovery.captureFailed()
|
|
491
|
+
FileHandle.standardError.write(
|
|
492
|
+
"simframed: capture failed: \(error) (\(recovery.consecutiveFailures) in a row)\n".data(using: .utf8)!)
|
|
493
|
+
// The display port can be torn down and rebuilt under a
|
|
494
|
+
// running daemon, and every read on the old descriptor
|
|
495
|
+
// returns nil from then on. Observed on a device that was
|
|
496
|
+
// awake and visible the whole time: six minutes of
|
|
497
|
+
// "the display surface could not be read", cured instantly
|
|
498
|
+
// by restarting the daemon. Reporting a failure loudly is
|
|
499
|
+
// right; never recovering from it is not, so re-resolve the
|
|
500
|
+
// port and re-arm the damage callback.
|
|
501
|
+
if due {
|
|
502
|
+
let onDamage = { lock.lock(); dirty = true; lock.unlock() }
|
|
503
|
+
// Escalate rather than repeat. Two successful re-resolves
|
|
504
|
+
// with no frame between them means the port was never the
|
|
505
|
+
// problem, so try the thing that until now needed a human:
|
|
506
|
+
// rebind to the device, which is what restarting the
|
|
507
|
+
// daemon did.
|
|
508
|
+
let rebinding = recovery.needsRebind
|
|
509
|
+
let outcome = rebinding
|
|
510
|
+
? recovery.rebind(platform: platform, udid: device.udid, onDamage: onDamage)
|
|
511
|
+
: recovery.reattach(platform: platform, onDamage: onDamage)
|
|
512
|
+
let what = rebinding ? "rebound to the device" : "re-resolved the display port"
|
|
513
|
+
switch outcome {
|
|
514
|
+
case .success(let after):
|
|
515
|
+
lock.lock(); dirty = true; lock.unlock()
|
|
516
|
+
FileHandle.standardError.write(
|
|
517
|
+
"simframed: \(what) after \(after) failed reads\n".data(using: .utf8)!)
|
|
518
|
+
case .failure(let error):
|
|
519
|
+
FileHandle.standardError.write(
|
|
520
|
+
"simframed: could not \(rebinding ? "rebind to the device" : "re-resolve the display port"): \(error)\n".data(using: .utf8)!)
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
// Say that capture is wedged rather than merely slow.
|
|
524
|
+
//
|
|
525
|
+
// The loop now tries two things — re-resolve the port, then
|
|
526
|
+
// rebind the device — and stops there. Restarting the device
|
|
527
|
+
// remains the user's to make: a capture loop that rebooted
|
|
528
|
+
// the device it was watching would be a tool reaching for
|
|
529
|
+
// the mains because a reading looked wrong. So it is published, `doctor` grades it and
|
|
530
|
+
// `simframe state` prints it, and an agent reads "the
|
|
531
|
+
// simulator is wedged" instead of "nothing changed".
|
|
532
|
+
if recovery.isStalled {
|
|
533
|
+
if stalledSince == nil { stalledSince = FrameStore.nowMs() }
|
|
534
|
+
try? store.writeCaptureHealth([
|
|
535
|
+
"stalled": true,
|
|
536
|
+
"since": stalledSince ?? FrameStore.nowMs(),
|
|
537
|
+
"at": FrameStore.nowMs(),
|
|
538
|
+
"consecutiveFailures": recovery.consecutiveFailures,
|
|
539
|
+
"reattaches": recovery.reattaches,
|
|
540
|
+
"reason": "\(error)",
|
|
541
|
+
])
|
|
542
|
+
}
|
|
543
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
473
544
|
}
|
|
474
|
-
// Say that capture is wedged rather than merely slow, and
|
|
475
|
-
// then do nothing about it. The cure for this state is a
|
|
476
|
-
// device restart, which is the user's to make: a capture
|
|
477
|
-
// loop that rebooted the device it was watching would be a
|
|
478
|
-
// tool reaching for the mains because a reading looked
|
|
479
|
-
// wrong. So it is published, `doctor` grades it and
|
|
480
|
-
// `simframe state` prints it, and an agent reads "the
|
|
481
|
-
// simulator is wedged" instead of "nothing changed".
|
|
482
|
-
if recovery.isStalled {
|
|
483
|
-
if stalledSince == nil { stalledSince = FrameStore.nowMs() }
|
|
484
|
-
try? store.writeCaptureHealth([
|
|
485
|
-
"stalled": true,
|
|
486
|
-
"since": stalledSince ?? FrameStore.nowMs(),
|
|
487
|
-
"at": FrameStore.nowMs(),
|
|
488
|
-
"consecutiveFailures": recovery.consecutiveFailures,
|
|
489
|
-
"reattaches": recovery.reattaches,
|
|
490
|
-
"reason": "\(error)",
|
|
491
|
-
])
|
|
492
|
-
}
|
|
493
|
-
Thread.sleep(forTimeInterval: 0.5)
|
|
494
545
|
}
|
|
495
546
|
}
|
|
496
547
|
|
|
@@ -499,8 +550,12 @@ case "run":
|
|
|
499
550
|
let sorted = latencies.sorted()
|
|
500
551
|
let median = sorted.isEmpty ? 0 : sorted[sorted.count / 2]
|
|
501
552
|
FileHandle.standardError.write(
|
|
502
|
-
String(
|
|
503
|
-
|
|
553
|
+
String(
|
|
554
|
+
format: "simframed: %.1f fps, median %.2fms, rss %.0fMB\n",
|
|
555
|
+
Double(frames) / (wall - lastReport),
|
|
556
|
+
median,
|
|
557
|
+
Double(residentBytes()) / 1_048_576
|
|
558
|
+
).data(using: .utf8)!)
|
|
504
559
|
frames = 0; latencies.removeAll(); lastReport = wall
|
|
505
560
|
}
|
|
506
561
|
|
|
@@ -332,6 +332,46 @@ final class CaptureRecoveryTests: XCTestCase {
|
|
|
332
332
|
XCTAssertEqual(recovery.reattaches, 0)
|
|
333
333
|
}
|
|
334
334
|
|
|
335
|
+
func testTwoDeadReResolvesEscalateToRebindingTheDevice() {
|
|
336
|
+
// The cure that used to require a human. Two successful re-resolves
|
|
337
|
+
// with no frame between them says the port was never the problem, so
|
|
338
|
+
// the next attempt rebinds the device itself — which is what
|
|
339
|
+
// restarting the daemon did, and it was the only known cure for four
|
|
340
|
+
// wedges in one afternoon.
|
|
341
|
+
let platform = StubPlatform()
|
|
342
|
+
_ = try? platform.attach(udid: "STUB-1")
|
|
343
|
+
var recovery = CaptureRecovery(threshold: 1)
|
|
344
|
+
|
|
345
|
+
XCTAssertFalse(recovery.needsRebind, "a healthy loop rebinds nothing")
|
|
346
|
+
_ = recovery.captureFailed()
|
|
347
|
+
_ = recovery.reattach(platform: platform, onDamage: {})
|
|
348
|
+
XCTAssertFalse(recovery.needsRebind, "one re-resolve deserves the benefit of the doubt")
|
|
349
|
+
_ = recovery.captureFailed()
|
|
350
|
+
_ = recovery.reattach(platform: platform, onDamage: {})
|
|
351
|
+
XCTAssertTrue(recovery.needsRebind, "two is enough")
|
|
352
|
+
|
|
353
|
+
var damaged = false
|
|
354
|
+
let outcome = recovery.rebind(platform: platform, udid: "STUB-1") { damaged = true }
|
|
355
|
+
guard case .success = outcome else { return XCTFail("rebind should succeed on a live stub") }
|
|
356
|
+
XCTAssertEqual(platform.rebindCount, 1, "it rebound the device, not the port")
|
|
357
|
+
XCTAssertEqual(platform.reattachCount, 2, "and did not re-resolve a third time")
|
|
358
|
+
XCTAssertEqual(recovery.consecutiveFailures, 0)
|
|
359
|
+
|
|
360
|
+
// Still stalled: a rebind is an attempt, not evidence. Only a frame is.
|
|
361
|
+
XCTAssertTrue(recovery.isStalled, "the reattach count is untouched by an attempt")
|
|
362
|
+
platform.simulateChange()
|
|
363
|
+
XCTAssertTrue(damaged, "the damage callback was re-armed on the new port")
|
|
364
|
+
|
|
365
|
+
// Bounded, so a wedged device is not rebound every half second forever.
|
|
366
|
+
_ = recovery.rebind(platform: platform, udid: "STUB-1", onDamage: {})
|
|
367
|
+
XCTAssertFalse(recovery.needsRebind, "two attempts per stall episode is the cap")
|
|
368
|
+
|
|
369
|
+
// And a real frame resets everything, including the rebind budget.
|
|
370
|
+
recovery.captureSucceeded()
|
|
371
|
+
XCTAssertFalse(recovery.isStalled)
|
|
372
|
+
XCTAssertEqual(recovery.rebinds, 0)
|
|
373
|
+
}
|
|
374
|
+
|
|
335
375
|
func testAReattachThatKeepsFailingIsAlsoAStall() {
|
|
336
376
|
// The other direction: when the re-resolve itself fails the count does
|
|
337
377
|
// keep growing, because nothing resets it.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "simframe",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"mcpName": "io.github.lvlrSajjad/simframe",
|
|
5
5
|
"description": "Always-warm iOS Simulator and Android emulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
|
|
6
6
|
"keywords": [
|
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
},
|
|
43
43
|
"files": [
|
|
44
44
|
"src",
|
|
45
|
+
"flows",
|
|
45
46
|
"native/ocr.swift",
|
|
46
47
|
"native/simframed/Package.swift",
|
|
47
48
|
"native/simframed/Sources",
|