simframe 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -3
- package/flows/hpi-suite.json +68 -0
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +49 -1
- package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +7 -0
- package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +11 -0
- package/native/simframed/Sources/SimframeCore/CaptureRecovery.swift +48 -0
- package/native/simframed/Sources/simframed/main.swift +128 -73
- package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +40 -0
- package/package.json +2 -1
- package/scripts/bench-hpi.mjs +254 -0
- package/scripts/check-package.mjs +7 -0
- package/scripts/check-private.mjs +143 -0
- package/scripts/eval-perception.mjs +248 -0
- package/src/actions.js +333 -14
- package/src/analyze.js +70 -0
- package/src/baseline.js +333 -0
- package/src/cli.js +410 -4
- package/src/daemon.js +9 -0
- package/src/fingerprint.js +7 -1
- package/src/graph.js +262 -1
- package/src/index.js +335 -22
- package/src/input.js +155 -1
- package/src/intent.js +11 -2
- package/src/matching.js +136 -4
- package/src/mcp.js +14 -1
- package/src/metrics.js +596 -0
- package/src/navigate.js +47 -7
- package/src/platform/android.js +16 -1
- package/src/platform/index.js +3 -0
- package/src/platform/ios.js +40 -0
- package/src/screenmap.js +55 -14
- package/src/view.js +65 -3
package/README.md
CHANGED
|
@@ -255,10 +255,11 @@ nav-bar:
|
|
|
255
255
|
#2 text 201,64 Inbox
|
|
256
256
|
content:
|
|
257
257
|
#3 cell 201,140 Weekly digest
|
|
258
|
-
#4
|
|
258
|
+
#4 field 201,196 Search = weekly ~ weekly|
|
|
259
|
+
#5 switch 201,252 Notifications = 1
|
|
259
260
|
tab-bar:
|
|
260
|
-
#
|
|
261
|
-
#
|
|
261
|
+
#6 text 62,835 Inbox
|
|
262
|
+
#7 text 201,835 Settings
|
|
262
263
|
```
|
|
263
264
|
|
|
264
265
|
Region first, because "Inbox" the title and "Inbox" the tab differ only by where
|
|
@@ -267,6 +268,18 @@ is a selector: whatever this calls `#3`, the next call can tap as `#3` without
|
|
|
267
268
|
describing it. A ref is valid only while that screen is showing — used on a
|
|
268
269
|
different screen it refuses rather than tapping whatever now sits there.
|
|
269
270
|
|
|
271
|
+
`= something` is what the control *contains*, from the accessibility tree, and
|
|
272
|
+
`~ something` is what OCR read off the pixels. Both are printed, and where they
|
|
273
|
+
disagree that is the point: one is authoritative and the other is what is
|
|
274
|
+
actually on screen, and a field mid-edit can legitimately differ. A row with no
|
|
275
|
+
`=` is a control that reports no value, not an empty one.
|
|
276
|
+
|
|
277
|
+
When the elements were recalled from screen memory rather than looked at just
|
|
278
|
+
now, the header says so and how long ago — `elements recalled from 41s ago —
|
|
279
|
+
pass refresh for what is there now`. Identity is cached on purpose, because a
|
|
280
|
+
list with new rows is the same screen; contents are exactly what changes without
|
|
281
|
+
the screen changing, so the age is worth seeing.
|
|
282
|
+
|
|
270
283
|
Three ways to name a control, anywhere one is named:
|
|
271
284
|
|
|
272
285
|
| | |
|
|
@@ -541,6 +554,11 @@ simframe recall --ago=15000 # the frame from 15s ago
|
|
|
541
554
|
simframe frame --out=now.png # newest frame, native resolution, to a file
|
|
542
555
|
simframe strip --count=6 # contact sheet, for an animation
|
|
543
556
|
simframe doctor --strict # any degraded layer is a non-zero exit
|
|
557
|
+
simframe escalations # why simframe still needs a model, by reason
|
|
558
|
+
simframe escalations --session # ...this agent only, not every agent on the device
|
|
559
|
+
simframe hpi # speed and accuracy against a human baseline
|
|
560
|
+
simframe baseline record settings-larger-text --runs=5 # record the human
|
|
561
|
+
simframe input reset # rebuild the HID session, without restarting anything
|
|
544
562
|
simframe start / status / stop [--force] / devices
|
|
545
563
|
simframe ui --device=emulator-5554 # or export SIMFRAME_DEVICE once
|
|
546
564
|
```
|
|
@@ -643,6 +661,16 @@ said a word — the exact failure shape, found by the thing built to catch it.
|
|
|
643
661
|
dramatically between visits will simply be rebuilt.
|
|
644
662
|
- It speeds up *confirming* a fix, not *locating* one. A bug living in a memo
|
|
645
663
|
comparator or a stale closure is not visible in any frame.
|
|
664
|
+
- A switch is tapped at the centre of its frame, and a switch's frame is the
|
|
665
|
+
whole row — so the tap lands on the label and the control, which sits at the
|
|
666
|
+
trailing end, does not move. Use `@x,y` on the control for now. Filed with
|
|
667
|
+
the measurement in `docs/DEFERRED.md`; it is a role-specific tap point, not a
|
|
668
|
+
patch at one call site.
|
|
669
|
+
- The simulator's display pipeline stops rendering under rapid app relaunch —
|
|
670
|
+
about six cycles, reproducibly — and every frame comes back black while
|
|
671
|
+
`simctl` itself reports success. simframe now says so instead of reading a
|
|
672
|
+
black screen as a calm one, but it cannot fix it: restarting the device is
|
|
673
|
+
the cure that always works, and it usually recovers on its own.
|
|
646
674
|
|
|
647
675
|
## Roadmap
|
|
648
676
|
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"name": "settings-larger-text",
|
|
4
|
+
"note": "Four steps into a nested Settings list. The two Settings sublists are the fingerprint pair that needs a nav title to tell them apart, so this flow exercises screen identity as well as navigation.",
|
|
5
|
+
"minSteps": 4,
|
|
6
|
+
"endsOn": "Larger Text",
|
|
7
|
+
"human": [
|
|
8
|
+
"Tap Settings on the home screen.",
|
|
9
|
+
"Tap Accessibility.",
|
|
10
|
+
"Tap Display & Text Size.",
|
|
11
|
+
"Tap Larger Text, and stop there."
|
|
12
|
+
],
|
|
13
|
+
"reset": {
|
|
14
|
+
"terminate": [
|
|
15
|
+
"com.apple.Preferences"
|
|
16
|
+
],
|
|
17
|
+
"home": true,
|
|
18
|
+
"launch": "com.apple.Preferences",
|
|
19
|
+
"rootMarker": "Accessibility",
|
|
20
|
+
"maxBack": 4
|
|
21
|
+
},
|
|
22
|
+
"steps": [
|
|
23
|
+
{
|
|
24
|
+
"launch": {
|
|
25
|
+
"value": "com.apple.Preferences"
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"tap": "Accessibility"
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"tap": "Display & Text Size"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"tap": "Larger Text"
|
|
36
|
+
}
|
|
37
|
+
]
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"name": "contacts-kate-bell",
|
|
41
|
+
"note": "An indexed list with a search field, two steps deep. Short on purpose: a flow this shallow is where a per-step overhead shows up as a ratio rather than being absorbed by navigation.",
|
|
42
|
+
"minSteps": 2,
|
|
43
|
+
"endsOn": "Kate Bell",
|
|
44
|
+
"human": [
|
|
45
|
+
"Tap Contacts on the home screen.",
|
|
46
|
+
"Tap Kate Bell, and stop there."
|
|
47
|
+
],
|
|
48
|
+
"reset": {
|
|
49
|
+
"terminate": [
|
|
50
|
+
"com.apple.MobileAddressBook"
|
|
51
|
+
],
|
|
52
|
+
"home": true,
|
|
53
|
+
"launch": "com.apple.MobileAddressBook",
|
|
54
|
+
"rootMarker": "Kate Bell",
|
|
55
|
+
"maxBack": 3
|
|
56
|
+
},
|
|
57
|
+
"steps": [
|
|
58
|
+
{
|
|
59
|
+
"launch": {
|
|
60
|
+
"value": "com.apple.MobileAddressBook"
|
|
61
|
+
}
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"tap": "Kate Bell"
|
|
65
|
+
}
|
|
66
|
+
]
|
|
67
|
+
}
|
|
68
|
+
]
|
|
@@ -157,6 +157,21 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
|
|
|
157
157
|
return try resolveDisplay(on: device, warmInput: true)
|
|
158
158
|
}
|
|
159
159
|
|
|
160
|
+
/// Rebind to the device from scratch: a fresh device object, a fresh port.
|
|
161
|
+
///
|
|
162
|
+
/// `reattachDisplay` reuses the cached `device`, which is right for a port
|
|
163
|
+
/// that was rebuilt under a living device and wrong for everything else —
|
|
164
|
+
/// the device object itself can be stale after a restart, and re-walking
|
|
165
|
+
/// its `ioPorts` then re-finds the same dead descriptors. This asks
|
|
166
|
+
/// CoreSimulator for the device list again, so nothing from the previous
|
|
167
|
+
/// session survives. Input is warmed too, because a session that outlived
|
|
168
|
+
/// its device is dead anyway.
|
|
169
|
+
public func reattachDevice(udid: String?) throws -> DeviceInfo {
|
|
170
|
+
device = nil
|
|
171
|
+
display = nil
|
|
172
|
+
return try attach(udid: udid)
|
|
173
|
+
}
|
|
174
|
+
|
|
160
175
|
/// Re-resolve the display port on the device we are already bound to.
|
|
161
176
|
///
|
|
162
177
|
/// Input is deliberately left alone: the HID session is independent of the
|
|
@@ -180,15 +195,36 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
|
|
|
180
195
|
let sizeSel = NSSelectorFromString("displaySize")
|
|
181
196
|
typealias SizeFn = @convention(c) (AnyObject, Selector) -> CGSize
|
|
182
197
|
|
|
198
|
+
// Two passes, because a size is a claim and a surface is evidence.
|
|
199
|
+
//
|
|
200
|
+
// A torn-down port keeps reporting a real `displaySize` while
|
|
201
|
+
// `framebufferSurface` returns nil for the rest of the device's life.
|
|
202
|
+
// Selecting on size alone therefore reattached to the dead port and
|
|
203
|
+
// declared success: the daemon logged "re-resolved the display port
|
|
204
|
+
// after 6 failed reads" and then failed six more, in a loop, until the
|
|
205
|
+
// device was restarted. That is the pathology CaptureRecovery's
|
|
206
|
+
// `stalledAfterReattaches` exists to *notice*; this is what fixes it.
|
|
207
|
+
//
|
|
208
|
+
// The size check stays as the cheap pre-filter. The second pass is the
|
|
209
|
+
// one that decides, and if no candidate yields a surface the first
|
|
210
|
+
// plausible one is used anyway — during boot the port is real and the
|
|
211
|
+
// surface has simply not arrived yet, and refusing to attach then
|
|
212
|
+
// would trade a recoverable wedge for a daemon that never starts.
|
|
213
|
+
var candidates: [NSObject] = []
|
|
183
214
|
for port in ports {
|
|
184
215
|
guard port.responds(to: descriptorSel),
|
|
185
216
|
let descriptor = port.perform(descriptorSel)?.takeUnretainedValue() as? NSObject,
|
|
186
217
|
descriptor.conforms(to: proto),
|
|
187
218
|
descriptor.responds(to: sizeSel),
|
|
188
219
|
let sizeImp = descriptor.method(for: sizeSel) else { continue }
|
|
189
|
-
// Several ports conform; only the live one reports a real size.
|
|
190
220
|
let size = unsafeBitCast(sizeImp, to: SizeFn.self)(descriptor, sizeSel)
|
|
191
221
|
guard size.width > 0, size.height > 0 else { continue }
|
|
222
|
+
candidates.append(descriptor)
|
|
223
|
+
}
|
|
224
|
+
let live = candidates.first(where: Self.yieldsSurface) ?? candidates.first
|
|
225
|
+
for descriptor in candidates where descriptor === live {
|
|
226
|
+
let sizeImp = descriptor.method(for: sizeSel)!
|
|
227
|
+
let size = unsafeBitCast(sizeImp, to: SizeFn.self)(descriptor, sizeSel)
|
|
192
228
|
display = descriptor
|
|
193
229
|
// The bridge captures one device's token and installs itself on a
|
|
194
230
|
// process-wide translator, so it belongs to the device it was built
|
|
@@ -259,6 +295,18 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
|
|
|
259
295
|
}
|
|
260
296
|
}
|
|
261
297
|
|
|
298
|
+
/// Does this display descriptor actually produce a framebuffer?
|
|
299
|
+
///
|
|
300
|
+
/// The single question that separates a live port from a torn-down one, and
|
|
301
|
+
/// it is not the one the resolver used to ask. Cheap: one selector call and
|
|
302
|
+
/// a cast, no lock, no pixels read.
|
|
303
|
+
static func yieldsSurface(_ descriptor: NSObject) -> Bool {
|
|
304
|
+
guard descriptor.responds(to: NSSelectorFromString("framebufferSurface")),
|
|
305
|
+
let raw = descriptor.perform(NSSelectorFromString("framebufferSurface"))?.takeUnretainedValue()
|
|
306
|
+
else { return false }
|
|
307
|
+
return raw is IOSurface
|
|
308
|
+
}
|
|
309
|
+
|
|
262
310
|
public func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T {
|
|
263
311
|
guard let display else { throw PrivateAPIError.noDisplayPort }
|
|
264
312
|
guard let raw = display.perform(NSSelectorFromString("framebufferSurface"))?.takeUnretainedValue(),
|
|
@@ -84,6 +84,13 @@ public protocol SimulatorPlatform: AnyObject {
|
|
|
84
84
|
/// perfectly visible, and a daemon restart fixed it instantly. Without a
|
|
85
85
|
/// way to re-resolve, a restart is the only cure.
|
|
86
86
|
func reattachDisplay() throws -> DeviceInfo
|
|
87
|
+
/// Rebind from scratch: a fresh device object as well as a fresh port.
|
|
88
|
+
///
|
|
89
|
+
/// The escalation for when re-resolving the port has demonstrably not
|
|
90
|
+
/// helped. `reattachDisplay` re-walks the cached device's ports, which
|
|
91
|
+
/// re-finds the same dead descriptors when it is the device binding that
|
|
92
|
+
/// is stale.
|
|
93
|
+
func reattachDevice(udid: String?) throws -> DeviceInfo
|
|
87
94
|
/// Borrow the current framebuffer. The pointer is only valid inside `body`.
|
|
88
95
|
func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T
|
|
89
96
|
/// Called whenever the display reports damage — the per-redraw signal, so a
|
|
@@ -33,6 +33,10 @@ public final class StubPlatform: SimulatorPlatform {
|
|
|
33
33
|
/// Counted so a test can assert the capture loop actually tries to recover
|
|
34
34
|
/// rather than logging the same failure forever.
|
|
35
35
|
public private(set) var reattachCount = 0
|
|
36
|
+
/// Full rebinds, counted separately: the escalation is a different act
|
|
37
|
+
/// from re-resolving a port and a test that cannot tell them apart cannot
|
|
38
|
+
/// assert the escalation happened.
|
|
39
|
+
public private(set) var rebindCount = 0
|
|
36
40
|
private var attachedUdid: String?
|
|
37
41
|
|
|
38
42
|
/// Counted, so a test can assert that a dead input path is actually retried.
|
|
@@ -50,6 +54,13 @@ public final class StubPlatform: SimulatorPlatform {
|
|
|
50
54
|
return DeviceInfo(udid: attachedUdid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
|
|
51
55
|
}
|
|
52
56
|
|
|
57
|
+
public func reattachDevice(udid: String?) throws -> DeviceInfo {
|
|
58
|
+
rebindCount += 1
|
|
59
|
+
if failReattach { throw PrivateAPIError.noDisplayPort }
|
|
60
|
+
attachedUdid = udid ?? attachedUdid
|
|
61
|
+
return DeviceInfo(udid: attachedUdid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
|
|
62
|
+
}
|
|
63
|
+
|
|
53
64
|
public func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T {
|
|
54
65
|
let bytesPerRow = width * 4 + 40 // deliberate padding: mirrors real surfaces
|
|
55
66
|
if buffer.count != bytesPerRow * height {
|
|
@@ -36,9 +36,19 @@ public struct CaptureRecovery {
|
|
|
36
36
|
/// failure count does keep growing because nothing resets it.
|
|
37
37
|
public static let stalledAfterFailures = reattachAfterFailures * 3
|
|
38
38
|
|
|
39
|
+
/// How many times to rebind the device per stall episode.
|
|
40
|
+
///
|
|
41
|
+
/// Bounded because a rebind asks CoreSimulator for the whole device list
|
|
42
|
+
/// and warms input: worth doing when re-resolving has failed twice, not
|
|
43
|
+
/// worth doing every half second forever. Two attempts, then the loop goes
|
|
44
|
+
/// back to reporting the state it is in.
|
|
45
|
+
public static let maxRebinds = 2
|
|
46
|
+
|
|
39
47
|
public private(set) var consecutiveFailures = 0
|
|
40
48
|
/// Successful re-resolves since the last real frame.
|
|
41
49
|
public private(set) var reattaches = 0
|
|
50
|
+
/// Full rebinds since the last real frame.
|
|
51
|
+
public private(set) var rebinds = 0
|
|
42
52
|
private let threshold: Int
|
|
43
53
|
|
|
44
54
|
public init(threshold: Int = CaptureRecovery.reattachAfterFailures) {
|
|
@@ -54,12 +64,24 @@ public struct CaptureRecovery {
|
|
|
54
64
|
reattaches >= Self.stalledAfterReattaches || consecutiveFailures >= Self.stalledAfterFailures
|
|
55
65
|
}
|
|
56
66
|
|
|
67
|
+
/// Has re-resolving the port had its chance?
|
|
68
|
+
///
|
|
69
|
+
/// Two successful re-resolves with no frame between them is the port
|
|
70
|
+
/// telling us it was never the problem. That was already the *stalled*
|
|
71
|
+
/// signal; now it is also the trigger to try the one thing that had only
|
|
72
|
+
/// ever been done by hand — rebinding to the device, which is what
|
|
73
|
+
/// restarting the daemon did.
|
|
74
|
+
public var needsRebind: Bool {
|
|
75
|
+
reattaches >= Self.stalledAfterReattaches && rebinds < Self.maxRebinds
|
|
76
|
+
}
|
|
77
|
+
|
|
57
78
|
public mutating func captureSucceeded() {
|
|
58
79
|
consecutiveFailures = 0
|
|
59
80
|
// A real frame is the only evidence that health is back. Resetting this
|
|
60
81
|
// anywhere else — on a re-resolve, say — is how the loop above stayed
|
|
61
82
|
// invisible.
|
|
62
83
|
reattaches = 0
|
|
84
|
+
rebinds = 0
|
|
63
85
|
}
|
|
64
86
|
|
|
65
87
|
/// Records a failure and says whether the port is now due a re-resolve.
|
|
@@ -91,4 +113,30 @@ public struct CaptureRecovery {
|
|
|
91
113
|
return .failure(error)
|
|
92
114
|
}
|
|
93
115
|
}
|
|
116
|
+
|
|
117
|
+
/// Rebind to the device itself, and re-arm the callback on the new port.
|
|
118
|
+
///
|
|
119
|
+
/// The escalation `needsRebind` gates. Re-arming matters here for the same
|
|
120
|
+
/// reason it does in `reattach`: a fresh descriptor with no callback on it
|
|
121
|
+
/// is a daemon that has recovered and will never notice another change,
|
|
122
|
+
/// which looks exactly like the failure it just recovered from.
|
|
123
|
+
public mutating func rebind(
|
|
124
|
+
platform: SimulatorPlatform,
|
|
125
|
+
udid: String?,
|
|
126
|
+
onDamage: @escaping () -> Void
|
|
127
|
+
) -> Result<Int, Error> {
|
|
128
|
+
let failures = consecutiveFailures
|
|
129
|
+
rebinds += 1
|
|
130
|
+
do {
|
|
131
|
+
_ = try platform.reattachDevice(udid: udid)
|
|
132
|
+
try platform.observeChanges(onDamage)
|
|
133
|
+
consecutiveFailures = 0
|
|
134
|
+
// `reattaches` is deliberately left alone. It is the evidence that
|
|
135
|
+
// the port was not the problem, and a rebind does not make that
|
|
136
|
+
// untrue — only a real frame does, in captureSucceeded().
|
|
137
|
+
return .success(failures)
|
|
138
|
+
} catch {
|
|
139
|
+
return .failure(error)
|
|
140
|
+
}
|
|
141
|
+
}
|
|
94
142
|
}
|
|
@@ -3,6 +3,25 @@ import Foundation
|
|
|
3
3
|
import PrivateAPI
|
|
4
4
|
import SimframeCore
|
|
5
5
|
|
|
6
|
+
/// Resident size of this process, in bytes, or 0 if the kernel will not say.
|
|
7
|
+
///
|
|
8
|
+
/// Logged with throughput because the capture wedge has no established cause
|
|
9
|
+
/// and memory pressure is one of two candidates. A number in the log every
|
|
10
|
+
/// second is what lets the next wedge be correlated with a spike — or clear
|
|
11
|
+
/// memory of suspicion, which is just as useful.
|
|
12
|
+
func residentBytes() -> UInt64 {
|
|
13
|
+
var info = mach_task_basic_info()
|
|
14
|
+
var count = mach_msg_type_number_t(
|
|
15
|
+
MemoryLayout<mach_task_basic_info>.size / MemoryLayout<natural_t>.size
|
|
16
|
+
)
|
|
17
|
+
let result = withUnsafeMutablePointer(to: &info) { pointer in
|
|
18
|
+
pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { rebound in
|
|
19
|
+
task_info(mach_task_self_, task_flavor_t(MACH_TASK_BASIC_INFO), rebound, &count)
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return result == KERN_SUCCESS ? info.resident_size : 0
|
|
23
|
+
}
|
|
24
|
+
|
|
6
25
|
let args = Array(CommandLine.arguments.dropFirst())
|
|
7
26
|
func flag(_ name: String) -> String? {
|
|
8
27
|
guard let a = args.first(where: { $0.hasPrefix("--\(name)=") }) else { return nil }
|
|
@@ -405,6 +424,15 @@ case "run":
|
|
|
405
424
|
}
|
|
406
425
|
defer { control.stop() }
|
|
407
426
|
|
|
427
|
+
// A new capture session inherits no stall.
|
|
428
|
+
//
|
|
429
|
+
// capture-health.json is cleared when a stalled loop captures a frame
|
|
430
|
+
// again, and a loop that dies while stalled never gets to. So a fresh
|
|
431
|
+
// daemon captured happily while `doctor` reported "stalled for 221s,
|
|
432
|
+
// 60 re-attaches" from its dead predecessor — and the display probe,
|
|
433
|
+
// correctly on that input, called it a simframe bug. It was: this one.
|
|
434
|
+
try? store.writeCaptureHealth(nil)
|
|
435
|
+
|
|
408
436
|
FileHandle.standardError.write("simframed: capturing \(device.name) (\(device.udid))\n".data(using: .utf8)!)
|
|
409
437
|
|
|
410
438
|
while true {
|
|
@@ -417,80 +445,103 @@ case "run":
|
|
|
417
445
|
if due {
|
|
418
446
|
lock.lock(); dirty = false; lock.unlock()
|
|
419
447
|
let t0 = DispatchTime.now().uptimeNanoseconds
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
// right; never recovering from it is not, so re-resolve the
|
|
459
|
-
// port and re-arm the damage callback.
|
|
460
|
-
if due {
|
|
461
|
-
switch recovery.reattach(platform: platform, onDamage: {
|
|
462
|
-
lock.lock(); dirty = true; lock.unlock()
|
|
463
|
-
}) {
|
|
464
|
-
case .success(let after):
|
|
465
|
-
lock.lock(); dirty = true; lock.unlock()
|
|
466
|
-
FileHandle.standardError.write(
|
|
467
|
-
"simframed: re-resolved the display port after \(after) failed reads\n"
|
|
468
|
-
.data(using: .utf8)!)
|
|
469
|
-
case .failure(let error):
|
|
448
|
+
// One pool per capture.
|
|
449
|
+
//
|
|
450
|
+
// There was none anywhere in this loop, and on Darwin that is
|
|
451
|
+
// the standard way to get the working set this daemon had:
|
|
452
|
+
// 732 MB resident after eleven minutes and 2831 frames, for a
|
|
453
|
+
// process that holds one frame at a time. CoreGraphics scaling
|
|
454
|
+
// and IOSurface access both produce autoreleased temporaries,
|
|
455
|
+
// and a `while true` loop with no pool of its own drains
|
|
456
|
+
// nothing. Whether that pressure is what wedges capture is
|
|
457
|
+
// unproven — the RSS also *fell* 200 MB in 25 s, so nothing is
|
|
458
|
+
// leaking monotonically — but a frame grabber should not hold
|
|
459
|
+
// half a gigabyte either way.
|
|
460
|
+
autoreleasepool {
|
|
461
|
+
do {
|
|
462
|
+
// One grab produces both sizes; the surface pointer is only
|
|
463
|
+
// valid inside this call, so nothing may be deferred out of it.
|
|
464
|
+
let wantFull = store.wantsFullFrame()
|
|
465
|
+
let (bmp, full) = try platform.withFrame { frame -> (Bitmap, Bitmap?) in
|
|
466
|
+
let scaled = CoreGraphicsScaler.bitmap(from: frame, targetLongEdge: longEdge)
|
|
467
|
+
?? Bitmap.from(frame, targetLongEdge: longEdge)
|
|
468
|
+
let native = wantFull
|
|
469
|
+
? CoreGraphicsScaler.bitmap(from: frame, targetLongEdge: max(frame.width, frame.height))
|
|
470
|
+
: nil
|
|
471
|
+
return (scaled, native)
|
|
472
|
+
}
|
|
473
|
+
let ms = Double(DispatchTime.now().uptimeNanoseconds - t0) / 1e6
|
|
474
|
+
try store.record(bmp, fullBitmap: full, captureMs: ms)
|
|
475
|
+
latencies.append(Double(DispatchTime.now().uptimeNanoseconds - t0) / 1e6)
|
|
476
|
+
frames += 1
|
|
477
|
+
lastCapture = now
|
|
478
|
+
let wasStalled = recovery.isStalled
|
|
479
|
+
recovery.captureSucceeded()
|
|
480
|
+
if wasStalled {
|
|
481
|
+
// A frame after a stall is the only thing that clears
|
|
482
|
+
// it, and it is worth saying out loud: the device came
|
|
483
|
+
// back on its own, which nobody would otherwise know.
|
|
484
|
+
try? store.writeCaptureHealth(nil)
|
|
485
|
+
stalledSince = nil
|
|
470
486
|
FileHandle.standardError.write(
|
|
471
|
-
"simframed:
|
|
487
|
+
"simframed: capture recovered on its own\n".data(using: .utf8)!)
|
|
472
488
|
}
|
|
489
|
+
} catch {
|
|
490
|
+
let due = recovery.captureFailed()
|
|
491
|
+
FileHandle.standardError.write(
|
|
492
|
+
"simframed: capture failed: \(error) (\(recovery.consecutiveFailures) in a row)\n".data(using: .utf8)!)
|
|
493
|
+
// The display port can be torn down and rebuilt under a
|
|
494
|
+
// running daemon, and every read on the old descriptor
|
|
495
|
+
// returns nil from then on. Observed on a device that was
|
|
496
|
+
// awake and visible the whole time: six minutes of
|
|
497
|
+
// "the display surface could not be read", cured instantly
|
|
498
|
+
// by restarting the daemon. Reporting a failure loudly is
|
|
499
|
+
// right; never recovering from it is not, so re-resolve the
|
|
500
|
+
// port and re-arm the damage callback.
|
|
501
|
+
if due {
|
|
502
|
+
let onDamage = { lock.lock(); dirty = true; lock.unlock() }
|
|
503
|
+
// Escalate rather than repeat. Two successful re-resolves
|
|
504
|
+
// with no frame between them means the port was never the
|
|
505
|
+
// problem, so try the thing that until now needed a human:
|
|
506
|
+
// rebind to the device, which is what restarting the
|
|
507
|
+
// daemon did.
|
|
508
|
+
let rebinding = recovery.needsRebind
|
|
509
|
+
let outcome = rebinding
|
|
510
|
+
? recovery.rebind(platform: platform, udid: device.udid, onDamage: onDamage)
|
|
511
|
+
: recovery.reattach(platform: platform, onDamage: onDamage)
|
|
512
|
+
let what = rebinding ? "rebound to the device" : "re-resolved the display port"
|
|
513
|
+
switch outcome {
|
|
514
|
+
case .success(let after):
|
|
515
|
+
lock.lock(); dirty = true; lock.unlock()
|
|
516
|
+
FileHandle.standardError.write(
|
|
517
|
+
"simframed: \(what) after \(after) failed reads\n".data(using: .utf8)!)
|
|
518
|
+
case .failure(let error):
|
|
519
|
+
FileHandle.standardError.write(
|
|
520
|
+
"simframed: could not \(rebinding ? "rebind to the device" : "re-resolve the display port"): \(error)\n".data(using: .utf8)!)
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
// Say that capture is wedged rather than merely slow.
|
|
524
|
+
//
|
|
525
|
+
// The loop now tries two things — re-resolve the port, then
|
|
526
|
+
// rebind the device — and stops there. Restarting the device
|
|
527
|
+
// remains the user's to make: a capture loop that rebooted
|
|
528
|
+
// the device it was watching would be a tool reaching for
|
|
529
|
+
// the mains because a reading looked wrong. So it is published, `doctor` grades it and
|
|
530
|
+
// `simframe state` prints it, and an agent reads "the
|
|
531
|
+
// simulator is wedged" instead of "nothing changed".
|
|
532
|
+
if recovery.isStalled {
|
|
533
|
+
if stalledSince == nil { stalledSince = FrameStore.nowMs() }
|
|
534
|
+
try? store.writeCaptureHealth([
|
|
535
|
+
"stalled": true,
|
|
536
|
+
"since": stalledSince ?? FrameStore.nowMs(),
|
|
537
|
+
"at": FrameStore.nowMs(),
|
|
538
|
+
"consecutiveFailures": recovery.consecutiveFailures,
|
|
539
|
+
"reattaches": recovery.reattaches,
|
|
540
|
+
"reason": "\(error)",
|
|
541
|
+
])
|
|
542
|
+
}
|
|
543
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
473
544
|
}
|
|
474
|
-
// Say that capture is wedged rather than merely slow, and
|
|
475
|
-
// then do nothing about it. The cure for this state is a
|
|
476
|
-
// device restart, which is the user's to make: a capture
|
|
477
|
-
// loop that rebooted the device it was watching would be a
|
|
478
|
-
// tool reaching for the mains because a reading looked
|
|
479
|
-
// wrong. So it is published, `doctor` grades it and
|
|
480
|
-
// `simframe state` prints it, and an agent reads "the
|
|
481
|
-
// simulator is wedged" instead of "nothing changed".
|
|
482
|
-
if recovery.isStalled {
|
|
483
|
-
if stalledSince == nil { stalledSince = FrameStore.nowMs() }
|
|
484
|
-
try? store.writeCaptureHealth([
|
|
485
|
-
"stalled": true,
|
|
486
|
-
"since": stalledSince ?? FrameStore.nowMs(),
|
|
487
|
-
"at": FrameStore.nowMs(),
|
|
488
|
-
"consecutiveFailures": recovery.consecutiveFailures,
|
|
489
|
-
"reattaches": recovery.reattaches,
|
|
490
|
-
"reason": "\(error)",
|
|
491
|
-
])
|
|
492
|
-
}
|
|
493
|
-
Thread.sleep(forTimeInterval: 0.5)
|
|
494
545
|
}
|
|
495
546
|
}
|
|
496
547
|
|
|
@@ -499,8 +550,12 @@ case "run":
|
|
|
499
550
|
let sorted = latencies.sorted()
|
|
500
551
|
let median = sorted.isEmpty ? 0 : sorted[sorted.count / 2]
|
|
501
552
|
FileHandle.standardError.write(
|
|
502
|
-
String(
|
|
503
|
-
|
|
553
|
+
String(
|
|
554
|
+
format: "simframed: %.1f fps, median %.2fms, rss %.0fMB\n",
|
|
555
|
+
Double(frames) / (wall - lastReport),
|
|
556
|
+
median,
|
|
557
|
+
Double(residentBytes()) / 1_048_576
|
|
558
|
+
).data(using: .utf8)!)
|
|
504
559
|
frames = 0; latencies.removeAll(); lastReport = wall
|
|
505
560
|
}
|
|
506
561
|
|
|
@@ -332,6 +332,46 @@ final class CaptureRecoveryTests: XCTestCase {
|
|
|
332
332
|
XCTAssertEqual(recovery.reattaches, 0)
|
|
333
333
|
}
|
|
334
334
|
|
|
335
|
+
func testTwoDeadReResolvesEscalateToRebindingTheDevice() {
|
|
336
|
+
// The cure that used to require a human. Two successful re-resolves
|
|
337
|
+
// with no frame between them says the port was never the problem, so
|
|
338
|
+
// the next attempt rebinds the device itself — which is what
|
|
339
|
+
// restarting the daemon did, and it was the only known cure for four
|
|
340
|
+
// wedges in one afternoon.
|
|
341
|
+
let platform = StubPlatform()
|
|
342
|
+
_ = try? platform.attach(udid: "STUB-1")
|
|
343
|
+
var recovery = CaptureRecovery(threshold: 1)
|
|
344
|
+
|
|
345
|
+
XCTAssertFalse(recovery.needsRebind, "a healthy loop rebinds nothing")
|
|
346
|
+
_ = recovery.captureFailed()
|
|
347
|
+
_ = recovery.reattach(platform: platform, onDamage: {})
|
|
348
|
+
XCTAssertFalse(recovery.needsRebind, "one re-resolve deserves the benefit of the doubt")
|
|
349
|
+
_ = recovery.captureFailed()
|
|
350
|
+
_ = recovery.reattach(platform: platform, onDamage: {})
|
|
351
|
+
XCTAssertTrue(recovery.needsRebind, "two is enough")
|
|
352
|
+
|
|
353
|
+
var damaged = false
|
|
354
|
+
let outcome = recovery.rebind(platform: platform, udid: "STUB-1") { damaged = true }
|
|
355
|
+
guard case .success = outcome else { return XCTFail("rebind should succeed on a live stub") }
|
|
356
|
+
XCTAssertEqual(platform.rebindCount, 1, "it rebound the device, not the port")
|
|
357
|
+
XCTAssertEqual(platform.reattachCount, 2, "and did not re-resolve a third time")
|
|
358
|
+
XCTAssertEqual(recovery.consecutiveFailures, 0)
|
|
359
|
+
|
|
360
|
+
// Still stalled: a rebind is an attempt, not evidence. Only a frame is.
|
|
361
|
+
XCTAssertTrue(recovery.isStalled, "the reattach count is untouched by an attempt")
|
|
362
|
+
platform.simulateChange()
|
|
363
|
+
XCTAssertTrue(damaged, "the damage callback was re-armed on the new port")
|
|
364
|
+
|
|
365
|
+
// Bounded, so a wedged device is not rebound every half second forever.
|
|
366
|
+
_ = recovery.rebind(platform: platform, udid: "STUB-1", onDamage: {})
|
|
367
|
+
XCTAssertFalse(recovery.needsRebind, "two attempts per stall episode is the cap")
|
|
368
|
+
|
|
369
|
+
// And a real frame resets everything, including the rebind budget.
|
|
370
|
+
recovery.captureSucceeded()
|
|
371
|
+
XCTAssertFalse(recovery.isStalled)
|
|
372
|
+
XCTAssertEqual(recovery.rebinds, 0)
|
|
373
|
+
}
|
|
374
|
+
|
|
335
375
|
func testAReattachThatKeepsFailingIsAlsoAStall() {
|
|
336
376
|
// The other direction: when the re-resolve itself fails the count does
|
|
337
377
|
// keep growing, because nothing resets it.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "simframe",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"mcpName": "io.github.lvlrSajjad/simframe",
|
|
5
5
|
"description": "Always-warm iOS Simulator and Android emulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
|
|
6
6
|
"keywords": [
|
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
},
|
|
43
43
|
"files": [
|
|
44
44
|
"src",
|
|
45
|
+
"flows",
|
|
45
46
|
"native/ocr.swift",
|
|
46
47
|
"native/simframed/Package.swift",
|
|
47
48
|
"native/simframed/Sources",
|