serve-sim 0.1.45 → 0.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Package.swift +8 -0
- package/README.md +2 -0
- package/Sources/SimNative/CaptureEngine.swift +23 -6
- package/Sources/SimNative/DeviceHubKeyboardBridge.swift +300 -0
- package/Sources/SimNative/FrameCapture.swift +87 -11
- package/Sources/SimNative/H264Encoder.swift +14 -10
- package/Sources/SimNative/HIDInjector.swift +19 -5
- package/Sources/SimNativeSupport/DeviceHubKeyboardRouting.swift +147 -0
- package/Sources/SimNativeSupport/FramebufferSurfaceSelector.swift +55 -0
- package/Tests/SimNativeSupportTests/DeviceHubKeyboardRoutingTests.swift +123 -0
- package/Tests/SimNativeSupportTests/FramebufferSurfaceSelectorTests.swift +68 -0
- package/dist/middleware.js +43 -43
- package/dist/native/serve-sim-native.node +0 -0
- package/dist/serve-sim.js +63 -63
- package/package.json +4 -1
- package/src/middleware.ts +5 -1
- package/src/native.ts +10 -10
- package/src/simctl.ts +11 -0
package/Package.swift
CHANGED
|
@@ -30,9 +30,13 @@ let package = Package(
|
|
|
30
30
|
.package(path: "node_modules/node-swift"),
|
|
31
31
|
],
|
|
32
32
|
targets: [
|
|
33
|
+
.target(
|
|
34
|
+
name: "SimNativeSupport"
|
|
35
|
+
),
|
|
33
36
|
.target(
|
|
34
37
|
name: "SimNative",
|
|
35
38
|
dependencies: [
|
|
39
|
+
"SimNativeSupport",
|
|
36
40
|
.product(name: "NodeAPI", package: "node-swift"),
|
|
37
41
|
.product(name: "NodeModuleSupport", package: "node-swift"),
|
|
38
42
|
],
|
|
@@ -46,6 +50,10 @@ let package = Package(
|
|
|
46
50
|
])
|
|
47
51
|
]
|
|
48
52
|
),
|
|
53
|
+
.testTarget(
|
|
54
|
+
name: "SimNativeSupportTests",
|
|
55
|
+
dependencies: ["SimNativeSupport"]
|
|
56
|
+
),
|
|
49
57
|
],
|
|
50
58
|
// The reused SimStreamHelper logic was written against the standalone helper
|
|
51
59
|
// (plain swiftc, which defaults to Swift 5 mode). Build in Swift 5 mode so its
|
package/README.md
CHANGED
|
@@ -36,6 +36,8 @@ Requires macOS with Xcode command line tools (`xcrun simctl`) and a [maintained
|
|
|
36
36
|
|
|
37
37
|
> **Note:** Apple Silicon (arm64) only. The bundled `serve-sim-bin` helper ships as an arm64 binary and does not run on Intel (x86_64) Macs.
|
|
38
38
|
|
|
39
|
+
> **Xcode 27 keyboard input:** Device Hub must be running with the target simulator window visible and frontmost. macOS may also require the app that launched `serve-sim` (for example Terminal) to be enabled in **System Settings → Privacy & Security → Accessibility**. Xcode 26 and older keep using the legacy simulator HID path. Set `SERVE_SIM_DISABLE_DEVICE_HUB_KEYBOARD=1` to opt out of the Xcode 27 bridge.
|
|
40
|
+
|
|
39
41
|
## CLI
|
|
40
42
|
|
|
41
43
|
```
|
|
@@ -48,7 +48,7 @@ actor CaptureConsumer<E: FrameEncoder>: CaptureConsuming {
|
|
|
48
48
|
let encoded = try await encoder.encode(frame)
|
|
49
49
|
await onFrame(encoded)
|
|
50
50
|
} catch {
|
|
51
|
-
print("error encoding frame: \(error)")
|
|
51
|
+
print("error encoding \(E.self) frame: \(error)")
|
|
52
52
|
continue
|
|
53
53
|
}
|
|
54
54
|
}
|
|
@@ -94,8 +94,25 @@ actor CaptureEngine {
|
|
|
94
94
|
// drop old frames if there's backpressure
|
|
95
95
|
bufferingPolicy: .bufferingNewest(1)
|
|
96
96
|
)
|
|
97
|
-
|
|
98
|
-
|
|
97
|
+
do {
|
|
98
|
+
try await frameCapture.start(deviceUDID: deviceUDID) { pixelBuffer, _ in
|
|
99
|
+
frameContinuation.yield(Frame(pixelBuffer: pixelBuffer))
|
|
100
|
+
}
|
|
101
|
+
} catch {
|
|
102
|
+
// Preserve retryability after an ordinary start failure, but never
|
|
103
|
+
// overwrite a concurrent stop() that moved the actor to .stopped.
|
|
104
|
+
frameContinuation.finish()
|
|
105
|
+
await frameCapture.stop()
|
|
106
|
+
if phase == .starting { phase = .unstarted }
|
|
107
|
+
throw error
|
|
108
|
+
}
|
|
109
|
+
// Actor methods are reentrant across the await above. A simulator can
|
|
110
|
+
// be shut down while FrameCapture is starting; do not resurrect a
|
|
111
|
+
// session that stop() already marked stopped.
|
|
112
|
+
guard phase == .starting else {
|
|
113
|
+
frameContinuation.finish()
|
|
114
|
+
await frameCapture.stop()
|
|
115
|
+
throw CancellationError()
|
|
99
116
|
}
|
|
100
117
|
Task {
|
|
101
118
|
for await frame in frames {
|
|
@@ -148,7 +165,7 @@ actor CaptureEngine {
|
|
|
148
165
|
let flagDescription: Int32 = 1 << 0
|
|
149
166
|
let flagKeyframe: Int32 = 1 << 1
|
|
150
167
|
|
|
151
|
-
guard let self else { return }
|
|
168
|
+
guard let self, let encoded else { return }
|
|
152
169
|
if let description = encoded.description {
|
|
153
170
|
await onFrame(
|
|
154
171
|
screenSize,
|
|
@@ -203,13 +220,13 @@ actor AVCCEncoder: FrameEncoder {
|
|
|
203
220
|
|
|
204
221
|
init() {}
|
|
205
222
|
|
|
206
|
-
func encode(_ frame: Frame) async throws -> H264Encoder.Encoded {
|
|
223
|
+
func encode(_ frame: Frame) async throws -> H264Encoder.Encoded? {
|
|
207
224
|
// TODO: cancel after timeout using TaskGroup
|
|
208
225
|
let result = try await h264Encoder.encode(
|
|
209
226
|
frame.pixelBuffer,
|
|
210
227
|
forceKeyframe: forceKeyframe,
|
|
211
228
|
)
|
|
212
|
-
forceKeyframe = false
|
|
229
|
+
if result != nil { forceKeyframe = false }
|
|
213
230
|
return result
|
|
214
231
|
}
|
|
215
232
|
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
import ApplicationServices
|
|
2
|
+
import AppKit
|
|
3
|
+
import CoreGraphics
|
|
4
|
+
import Foundation
|
|
5
|
+
import SimNativeSupport
|
|
6
|
+
|
|
7
|
+
/// Keyboard transport used by Xcode 27's Device Hub.
|
|
8
|
+
///
|
|
9
|
+
/// CoreSimulator still accepts legacy Indigo keyboard messages on Xcode 27,
|
|
10
|
+
/// but the iOS 27 guest no longer consumes them. Device Hub owns the new input
|
|
11
|
+
/// route, so this bridge posts ordinary macOS key events to its process and lets
|
|
12
|
+
/// the selected simulator window forward them. Xcode 26 and older never create
|
|
13
|
+
/// the bridge and continue using Indigo unchanged.
|
|
14
|
+
///
|
|
15
|
+
/// CGEvent can target an application, not one of its windows. To avoid typing
|
|
16
|
+
/// into a different simulator, the first key-down is accepted only when the
|
|
17
|
+
/// target device name identifies one visible, focused Device Hub window through
|
|
18
|
+
/// Accessibility. Accessibility titles remain available when CGWindow titles
|
|
19
|
+
/// are redacted without Screen Recording permission. The route remains latched
|
|
20
|
+
/// until every posted key is released, so one chord never splits between Device
|
|
21
|
+
/// Hub and the legacy Indigo fallback.
|
|
22
|
+
final class DeviceHubKeyboardBridge {
|
|
23
|
+
private static let bundleIdentifier = "com.apple.dt.Devices"
|
|
24
|
+
private static let minimumXcodeMajorVersion = 27
|
|
25
|
+
|
|
26
|
+
private struct ActiveTarget {
|
|
27
|
+
let processIdentifier: pid_t
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
private let expectedBundleURL: URL
|
|
31
|
+
private let deviceUDID: String
|
|
32
|
+
private let deviceName: String
|
|
33
|
+
private let targetWindowTitle: String
|
|
34
|
+
|
|
35
|
+
private var activeTarget: ActiveTarget?
|
|
36
|
+
private var pressedUsages = Set<UInt32>()
|
|
37
|
+
private var lastUnavailableReason: String?
|
|
38
|
+
|
|
39
|
+
private init(
|
|
40
|
+
expectedBundleURL: URL,
|
|
41
|
+
deviceUDID: String,
|
|
42
|
+
deviceName: String,
|
|
43
|
+
runtimeName: String
|
|
44
|
+
) {
|
|
45
|
+
self.expectedBundleURL = expectedBundleURL.resolvingSymlinksInPath().standardizedFileURL
|
|
46
|
+
self.deviceUDID = deviceUDID
|
|
47
|
+
self.deviceName = deviceName
|
|
48
|
+
self.targetWindowTitle = "\(deviceName) – \(runtimeName)"
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
static func makeIfSupported(
|
|
52
|
+
deviceUDID: String,
|
|
53
|
+
deviceName: String,
|
|
54
|
+
runtimeName: String?
|
|
55
|
+
) -> DeviceHubKeyboardBridge? {
|
|
56
|
+
let environment = ProcessInfo.processInfo.environment
|
|
57
|
+
guard !DeviceHubKeyboardConfiguration.isDisabled(
|
|
58
|
+
environmentValue: environment["SERVE_SIM_DISABLE_DEVICE_HUB_KEYBOARD"]
|
|
59
|
+
) else {
|
|
60
|
+
return nil
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
let developerURL = URL(fileURLWithPath: Xcode.developerDir(), isDirectory: true)
|
|
64
|
+
let xcodeURL = developerURL
|
|
65
|
+
.deletingLastPathComponent() // Contents
|
|
66
|
+
.deletingLastPathComponent() // Xcode.app
|
|
67
|
+
|
|
68
|
+
guard let bundle = Bundle(url: xcodeURL), xcodeMajorVersion(in: bundle) >= minimumXcodeMajorVersion else {
|
|
69
|
+
return nil
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
let deviceHubURL = xcodeURL
|
|
73
|
+
.appendingPathComponent("Contents", isDirectory: true)
|
|
74
|
+
.appendingPathComponent("Applications", isDirectory: true)
|
|
75
|
+
.appendingPathComponent("DeviceHub.app", isDirectory: true)
|
|
76
|
+
guard FileManager.default.fileExists(atPath: deviceHubURL.path) else { return nil }
|
|
77
|
+
|
|
78
|
+
guard let runtimeName, !runtimeName.isEmpty else { return nil }
|
|
79
|
+
return DeviceHubKeyboardBridge(
|
|
80
|
+
expectedBundleURL: deviceHubURL,
|
|
81
|
+
deviceUDID: deviceUDID,
|
|
82
|
+
deviceName: deviceName,
|
|
83
|
+
runtimeName: runtimeName
|
|
84
|
+
)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/// Returns true only when this event was posted through the guarded Device
|
|
88
|
+
/// Hub route. False lets HIDInjector use Indigo for backward compatibility
|
|
89
|
+
/// and for usages Device Hub cannot safely route.
|
|
90
|
+
func send(type: String, usage: UInt32) -> Bool {
|
|
91
|
+
let keyDown: Bool
|
|
92
|
+
switch type {
|
|
93
|
+
case "down": keyDown = true
|
|
94
|
+
case "up": keyDown = false
|
|
95
|
+
default: return false
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
guard let rawKeyCode = HIDKeyboardMapping.macVirtualKeyCode(for: usage) else { return false }
|
|
99
|
+
guard CGPreflightPostEventAccess() else {
|
|
100
|
+
return unavailable(
|
|
101
|
+
"macOS Accessibility permission is not granted "
|
|
102
|
+
+ "(System Settings > Privacy & Security > Accessibility)"
|
|
103
|
+
)
|
|
104
|
+
}
|
|
105
|
+
let keyCode = CGKeyCode(rawKeyCode)
|
|
106
|
+
|
|
107
|
+
let target: ActiveTarget
|
|
108
|
+
if keyDown {
|
|
109
|
+
if let activeTarget {
|
|
110
|
+
guard isExpectedDeviceHubRunning(processIdentifier: activeTarget.processIdentifier) else {
|
|
111
|
+
resetSequence()
|
|
112
|
+
return unavailable("Device Hub exited during a key sequence")
|
|
113
|
+
}
|
|
114
|
+
target = activeTarget
|
|
115
|
+
} else {
|
|
116
|
+
guard let resolved = resolveTarget() else { return false }
|
|
117
|
+
activeTarget = resolved
|
|
118
|
+
target = resolved
|
|
119
|
+
}
|
|
120
|
+
} else {
|
|
121
|
+
// An up event whose down used Indigo must stay on Indigo too.
|
|
122
|
+
guard pressedUsages.contains(usage), let activeTarget else { return false }
|
|
123
|
+
guard isExpectedDeviceHubRunning(processIdentifier: activeTarget.processIdentifier) else {
|
|
124
|
+
resetSequence()
|
|
125
|
+
return unavailable("Device Hub exited during a key sequence")
|
|
126
|
+
}
|
|
127
|
+
target = activeTarget
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
var nextPressedUsages = pressedUsages
|
|
131
|
+
if keyDown {
|
|
132
|
+
nextPressedUsages.insert(usage)
|
|
133
|
+
} else {
|
|
134
|
+
nextPressedUsages.remove(usage)
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
guard
|
|
138
|
+
let source = CGEventSource(stateID: .hidSystemState),
|
|
139
|
+
let event = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: keyDown)
|
|
140
|
+
else {
|
|
141
|
+
if pressedUsages.isEmpty { resetSequence() }
|
|
142
|
+
return unavailable("CoreGraphics could not create a keyboard event")
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
event.flags = Self.eventFlags(for: nextPressedUsages)
|
|
146
|
+
event.postToPid(target.processIdentifier)
|
|
147
|
+
|
|
148
|
+
pressedUsages = nextPressedUsages
|
|
149
|
+
lastUnavailableReason = nil
|
|
150
|
+
if pressedUsages.isEmpty { resetSequence() }
|
|
151
|
+
return true
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
private func resolveTarget() -> ActiveTarget? {
|
|
155
|
+
let applications = runningDeviceHubs()
|
|
156
|
+
guard applications.count == 1, let application = applications.first else {
|
|
157
|
+
let reason = applications.isEmpty
|
|
158
|
+
? "Device Hub from the selected Xcode is not running"
|
|
159
|
+
: "multiple matching Device Hub processes are running"
|
|
160
|
+
_ = unavailable(reason)
|
|
161
|
+
return nil
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
let processIdentifier = application.processIdentifier
|
|
165
|
+
let route = DeviceHubWindowRouter.route(
|
|
166
|
+
windows: Self.visibleWindows(processIdentifier: processIdentifier),
|
|
167
|
+
processIdentifier: processIdentifier,
|
|
168
|
+
targetWindowTitle: targetWindowTitle
|
|
169
|
+
)
|
|
170
|
+
switch route {
|
|
171
|
+
case .success:
|
|
172
|
+
return ActiveTarget(processIdentifier: processIdentifier)
|
|
173
|
+
case .failure(let failure):
|
|
174
|
+
_ = unavailable(failure.description)
|
|
175
|
+
return nil
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
private func runningDeviceHubs() -> [NSRunningApplication] {
|
|
180
|
+
NSRunningApplication
|
|
181
|
+
.runningApplications(withBundleIdentifier: Self.bundleIdentifier)
|
|
182
|
+
.filter { application in
|
|
183
|
+
guard !application.isTerminated, let bundleURL = application.bundleURL else { return false }
|
|
184
|
+
return bundleURL.resolvingSymlinksInPath().standardizedFileURL == expectedBundleURL
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
private func isExpectedDeviceHubRunning(processIdentifier: pid_t) -> Bool {
|
|
189
|
+
runningDeviceHubs().contains { $0.processIdentifier == processIdentifier }
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/// Return visible standard Device Hub windows with its key window first.
|
|
193
|
+
/// AX is already covered by the event-posting Accessibility permission and,
|
|
194
|
+
/// unlike CGWindowList, does not require Screen Recording to expose titles.
|
|
195
|
+
private static func visibleWindows(processIdentifier: pid_t) -> [DeviceHubWindow] {
|
|
196
|
+
let application = AXUIElementCreateApplication(processIdentifier)
|
|
197
|
+
guard
|
|
198
|
+
let windows: [AXUIElement] = accessibilityValue(
|
|
199
|
+
kAXWindowsAttribute,
|
|
200
|
+
from: application
|
|
201
|
+
),
|
|
202
|
+
let focusedWindow: AXUIElement = accessibilityValue(
|
|
203
|
+
kAXFocusedWindowAttribute,
|
|
204
|
+
from: application
|
|
205
|
+
),
|
|
206
|
+
windows.contains(where: { CFEqual($0, focusedWindow) }),
|
|
207
|
+
isVisibleStandardWindow(focusedWindow) == true
|
|
208
|
+
else {
|
|
209
|
+
return []
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
let orderedWindows = [focusedWindow] + windows.filter { !CFEqual($0, focusedWindow) }
|
|
213
|
+
var snapshot = [DeviceHubWindow]()
|
|
214
|
+
for (index, window) in orderedWindows.enumerated() {
|
|
215
|
+
// A partial AX snapshot is unsafe: dropping an unreadable focused
|
|
216
|
+
// window would make a different window appear to own keyboard focus.
|
|
217
|
+
guard
|
|
218
|
+
let isVisible = isVisibleStandardWindow(window),
|
|
219
|
+
let title: String = accessibilityValue(kAXTitleAttribute, from: window)
|
|
220
|
+
else { return [] }
|
|
221
|
+
guard isVisible else { continue }
|
|
222
|
+
snapshot.append(DeviceHubWindow(
|
|
223
|
+
processIdentifier: processIdentifier,
|
|
224
|
+
// The router only needs stable identity within this snapshot.
|
|
225
|
+
windowNumber: UInt32(index + 1),
|
|
226
|
+
name: title
|
|
227
|
+
))
|
|
228
|
+
}
|
|
229
|
+
return snapshot
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
private static func isVisibleStandardWindow(_ window: AXUIElement) -> Bool? {
|
|
233
|
+
guard
|
|
234
|
+
let role: String = accessibilityValue(kAXRoleAttribute, from: window),
|
|
235
|
+
let subrole: String = accessibilityValue(kAXSubroleAttribute, from: window),
|
|
236
|
+
let minimized: Bool = accessibilityValue(kAXMinimizedAttribute, from: window)
|
|
237
|
+
else { return nil }
|
|
238
|
+
return role == kAXWindowRole && subrole == kAXStandardWindowSubrole && !minimized
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
private static func accessibilityValue<T>(
|
|
242
|
+
_ attribute: String,
|
|
243
|
+
from element: AXUIElement
|
|
244
|
+
) -> T? {
|
|
245
|
+
var value: CFTypeRef?
|
|
246
|
+
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {
|
|
247
|
+
return nil
|
|
248
|
+
}
|
|
249
|
+
return value as? T
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
private func unavailable(_ reason: String) -> Bool {
|
|
253
|
+
if lastUnavailableReason != reason {
|
|
254
|
+
let shortUDID = String(deviceUDID.prefix(8))
|
|
255
|
+
print(
|
|
256
|
+
"[hid] Device Hub keyboard unavailable for \(deviceName) (\(shortUDID)): "
|
|
257
|
+
+ "\(reason); using legacy HID"
|
|
258
|
+
)
|
|
259
|
+
lastUnavailableReason = reason
|
|
260
|
+
}
|
|
261
|
+
return false
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
private func resetSequence() {
|
|
265
|
+
activeTarget = nil
|
|
266
|
+
pressedUsages.removeAll()
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
private static func eventFlags(for usages: Set<UInt32>) -> CGEventFlags {
|
|
270
|
+
var flags: CGEventFlags = []
|
|
271
|
+
for usage in usages {
|
|
272
|
+
switch HIDKeyboardMapping.modifier(for: usage) {
|
|
273
|
+
case .control: flags.insert(.maskControl)
|
|
274
|
+
case .shift: flags.insert(.maskShift)
|
|
275
|
+
case .option: flags.insert(.maskAlternate)
|
|
276
|
+
case .command: flags.insert(.maskCommand)
|
|
277
|
+
case nil: break
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return flags
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
private static func xcodeMajorVersion(in bundle: Bundle) -> Int {
|
|
284
|
+
if
|
|
285
|
+
let version = bundle.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String,
|
|
286
|
+
let major = Int(version.split(separator: ".").first ?? "")
|
|
287
|
+
{
|
|
288
|
+
return major
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
if let dtxcode = bundle.object(forInfoDictionaryKey: "DTXcode") as? String,
|
|
292
|
+
let value = Int(dtxcode) {
|
|
293
|
+
return value / 100
|
|
294
|
+
}
|
|
295
|
+
if let dtxcode = bundle.object(forInfoDictionaryKey: "DTXcode") as? NSNumber {
|
|
296
|
+
return dtxcode.intValue / 100
|
|
297
|
+
}
|
|
298
|
+
return 0
|
|
299
|
+
}
|
|
300
|
+
}
|
|
@@ -4,6 +4,7 @@ import CoreMedia
|
|
|
4
4
|
import CoreGraphics
|
|
5
5
|
import IOSurface
|
|
6
6
|
import ObjectiveC
|
|
7
|
+
import SimNativeSupport
|
|
7
8
|
|
|
8
9
|
/// Headless simulator frame capture via direct IOSurface access.
|
|
9
10
|
///
|
|
@@ -40,6 +41,9 @@ actor FrameCapture {
|
|
|
40
41
|
private var descriptors: [NSObject] = []
|
|
41
42
|
private var callbackUUIDs: [ObjectIdentifier: UUID] = [:]
|
|
42
43
|
private var ioClient: NSObject?
|
|
44
|
+
private var expectedScreenSize: FramebufferSurfaceSize?
|
|
45
|
+
private var didLogRejectedPresentationSurface = false
|
|
46
|
+
private var didLogMissingExpectedSurface = false
|
|
43
47
|
|
|
44
48
|
func start(deviceUDID: String, onFrame: @escaping @Sendable (CVPixelBuffer, CMTime) -> Void) throws {
|
|
45
49
|
self.onFrame = onFrame
|
|
@@ -53,6 +57,7 @@ actor FrameCapture {
|
|
|
53
57
|
guard state == "Booted" else {
|
|
54
58
|
throw makeError(2, "Device not booted (state: \(state))")
|
|
55
59
|
}
|
|
60
|
+
self.expectedScreenSize = Self.nativeScreenSize(for: device)
|
|
56
61
|
|
|
57
62
|
guard let io = device.perform(NSSelectorFromString("io"))?.takeUnretainedValue() as? NSObject else {
|
|
58
63
|
throw makeError(3, "Failed to get device IO")
|
|
@@ -139,22 +144,56 @@ actor FrameCapture {
|
|
|
139
144
|
return candidates
|
|
140
145
|
}
|
|
141
146
|
|
|
142
|
-
/// Return the descriptor
|
|
143
|
-
///
|
|
147
|
+
/// Return the descriptor matching the device type's native screen size.
|
|
148
|
+
/// Xcode 27 can expose an additional 7680x4320 presentation surface while
|
|
149
|
+
/// Device Hub is resizing; selecting the historical largest surface then
|
|
150
|
+
/// feeds a non-device frame to VideoToolbox and produces `encodingFailed`.
|
|
151
|
+
/// If screen metadata is unavailable, retain the old largest-live fallback.
|
|
144
152
|
private func pickBestDescriptor() -> NSObject? {
|
|
145
153
|
let surfSel = NSSelectorFromString("framebufferSurface")
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
154
|
+
let sizes = descriptors.map { desc -> FramebufferSurfaceSize in
|
|
155
|
+
guard let surfObj = desc.perform(surfSel)?.takeUnretainedValue() else {
|
|
156
|
+
return FramebufferSurfaceSize(width: 0, height: 0)
|
|
157
|
+
}
|
|
150
158
|
let surf = unsafeBitCast(surfObj, to: IOSurface.self)
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
159
|
+
return FramebufferSurfaceSize(
|
|
160
|
+
width: IOSurfaceGetWidth(surf),
|
|
161
|
+
height: IOSurfaceGetHeight(surf)
|
|
162
|
+
)
|
|
163
|
+
}
|
|
164
|
+
guard let selection = FramebufferSurfaceSelector.select(
|
|
165
|
+
from: sizes,
|
|
166
|
+
expectedSize: expectedScreenSize
|
|
167
|
+
) else {
|
|
168
|
+
return nil
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
if selection.matchedExpectedSize, !didLogRejectedPresentationSurface {
|
|
172
|
+
let selected = sizes[selection.index]
|
|
173
|
+
let selectedArea = Int64(selected.width) * Int64(selected.height)
|
|
174
|
+
if let larger = sizes.filter(\.isLive).max(by: {
|
|
175
|
+
Int64($0.width) * Int64($0.height) < Int64($1.width) * Int64($1.height)
|
|
176
|
+
}), Int64(larger.width) * Int64(larger.height) > selectedArea {
|
|
177
|
+
print(
|
|
178
|
+
"[capture] Ignoring non-device framebuffer \(larger.width)x\(larger.height); "
|
|
179
|
+
+ "native screen is \(selected.width)x\(selected.height)"
|
|
180
|
+
)
|
|
181
|
+
didLogRejectedPresentationSurface = true
|
|
155
182
|
}
|
|
183
|
+
} else if
|
|
184
|
+
!selection.matchedExpectedSize,
|
|
185
|
+
let expectedScreenSize,
|
|
186
|
+
!didLogMissingExpectedSurface
|
|
187
|
+
{
|
|
188
|
+
let selected = sizes[selection.index]
|
|
189
|
+
print(
|
|
190
|
+
"[capture] No framebuffer matches native screen "
|
|
191
|
+
+ "\(expectedScreenSize.width)x\(expectedScreenSize.height); "
|
|
192
|
+
+ "falling back to \(selected.width)x\(selected.height)"
|
|
193
|
+
)
|
|
194
|
+
didLogMissingExpectedSurface = true
|
|
156
195
|
}
|
|
157
|
-
return
|
|
196
|
+
return descriptors[selection.index]
|
|
158
197
|
}
|
|
159
198
|
|
|
160
199
|
// MARK: - Frame callbacks via objc_msgSend
|
|
@@ -270,7 +309,16 @@ actor FrameCapture {
|
|
|
270
309
|
callbackUUIDs.removeAll()
|
|
271
310
|
descriptors.removeAll()
|
|
272
311
|
lastSeeds.removeAll()
|
|
312
|
+
onFrame = nil
|
|
313
|
+
frameCount = 0
|
|
314
|
+
capturedWidth = 0
|
|
315
|
+
capturedHeight = 0
|
|
316
|
+
rewireTickCount = 0
|
|
317
|
+
lastCaptureTime = .now
|
|
273
318
|
ioClient = nil
|
|
319
|
+
expectedScreenSize = nil
|
|
320
|
+
didLogRejectedPresentationSurface = false
|
|
321
|
+
didLogMissingExpectedSurface = false
|
|
274
322
|
}
|
|
275
323
|
|
|
276
324
|
// MARK: - Helpers
|
|
@@ -280,6 +328,34 @@ actor FrameCapture {
|
|
|
280
328
|
userInfo: [NSLocalizedDescriptionKey: msg])
|
|
281
329
|
}
|
|
282
330
|
|
|
331
|
+
/// SimDeviceType.mainScreenSize is private but stable across the same
|
|
332
|
+
/// SimulatorKit versions already used by this file. Runtime validation on
|
|
333
|
+
/// Xcode 27 confirms that it reports native pixels, not logical points, and
|
|
334
|
+
/// matches the primary IOSurface exactly. The `@convention(c)` IMP type is
|
|
335
|
+
/// intentional: it preserves the platform CGSize return ABI (registers on
|
|
336
|
+
/// arm64 and the appropriate struct-return convention on x86_64).
|
|
337
|
+
private static func nativeScreenSize(for device: NSObject) -> FramebufferSurfaceSize? {
|
|
338
|
+
let deviceTypeSelector = NSSelectorFromString("deviceType")
|
|
339
|
+
guard
|
|
340
|
+
device.responds(to: deviceTypeSelector),
|
|
341
|
+
let deviceType = device.perform(deviceTypeSelector)?.takeUnretainedValue() as? NSObject
|
|
342
|
+
else {
|
|
343
|
+
return nil
|
|
344
|
+
}
|
|
345
|
+
let selector = NSSelectorFromString("mainScreenSize")
|
|
346
|
+
guard deviceType.responds(to: selector) else { return nil }
|
|
347
|
+
|
|
348
|
+
typealias GetSize = @convention(c) (AnyObject, Selector) -> CGSize
|
|
349
|
+
let getSize = unsafeBitCast(deviceType.method(for: selector), to: GetSize.self)
|
|
350
|
+
let size = getSize(deviceType, selector)
|
|
351
|
+
guard size.width.isFinite, size.height.isFinite else { return nil }
|
|
352
|
+
|
|
353
|
+
let width = Int(size.width.rounded())
|
|
354
|
+
let height = Int(size.height.rounded())
|
|
355
|
+
guard width > 0, height > 0 else { return nil }
|
|
356
|
+
return FramebufferSurfaceSize(width: width, height: height)
|
|
357
|
+
}
|
|
358
|
+
|
|
283
359
|
static func findSimDevice(udid: String) -> NSObject? {
|
|
284
360
|
guard let contextClass = NSClassFromString("SimServiceContext") as? NSObject.Type else { return nil }
|
|
285
361
|
let developerDir = Xcode.developerDir()
|
|
@@ -41,8 +41,9 @@ actor H264Encoder {
|
|
|
41
41
|
if let session { VTCompressionSessionInvalidate(session) }
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
///
|
|
45
|
-
|
|
44
|
+
/// Encode one frame. Real-time VideoToolbox sessions may deliberately drop
|
|
45
|
+
/// a frame under pressure; that is reported as `nil`, not as an error.
|
|
46
|
+
func encode(_ source: CVPixelBuffer, forceKeyframe: Bool = false) async throws -> Encoded? {
|
|
46
47
|
let w = Int32(CVPixelBufferGetWidth(source))
|
|
47
48
|
let h = Int32(CVPixelBufferGetHeight(source))
|
|
48
49
|
if session == nil || w != width || h != height {
|
|
@@ -60,7 +61,7 @@ actor H264Encoder {
|
|
|
60
61
|
? [kVTEncodeFrameOptionKey_ForceKeyFrame: kCFBooleanTrue!] as NSDictionary
|
|
61
62
|
: nil
|
|
62
63
|
|
|
63
|
-
let buffer: CMSampleBuffer
|
|
64
|
+
let result: (buffer: CMSampleBuffer?, status: OSStatus, flags: VTEncodeInfoFlags) = await withCheckedContinuation { continuation in
|
|
64
65
|
let status = VTCompressionSessionEncodeFrame(
|
|
65
66
|
session,
|
|
66
67
|
imageBuffer: source,
|
|
@@ -68,18 +69,20 @@ actor H264Encoder {
|
|
|
68
69
|
duration: .invalid,
|
|
69
70
|
frameProperties: frameProps,
|
|
70
71
|
infoFlagsOut: nil
|
|
71
|
-
) { @Sendable status,
|
|
72
|
-
guard status == noErr, let
|
|
73
|
-
continuation.resume(returning: nil)
|
|
72
|
+
) { @Sendable status, flags, sampleBuffer in
|
|
73
|
+
guard status == noErr, let sampleBuffer else {
|
|
74
|
+
continuation.resume(returning: (nil, status, flags))
|
|
74
75
|
return
|
|
75
76
|
}
|
|
76
|
-
continuation.resume(returning:
|
|
77
|
+
continuation.resume(returning: (sampleBuffer, status, flags))
|
|
77
78
|
}
|
|
78
79
|
if status != noErr {
|
|
79
|
-
continuation.resume(returning: nil)
|
|
80
|
+
continuation.resume(returning: (nil, status, []))
|
|
80
81
|
}
|
|
81
82
|
}
|
|
82
|
-
|
|
83
|
+
if result.status == noErr, result.flags.contains(.frameDropped) { return nil }
|
|
84
|
+
guard result.status == noErr else { throw Errors.encodingFailed(result.status) }
|
|
85
|
+
guard let buffer = result.buffer else { throw Errors.missingSampleBuffer }
|
|
83
86
|
return try extract(from: buffer)
|
|
84
87
|
}
|
|
85
88
|
|
|
@@ -219,7 +222,8 @@ actor H264Encoder {
|
|
|
219
222
|
|
|
220
223
|
enum Errors: Error {
|
|
221
224
|
case couldNotCreateSession
|
|
222
|
-
case encodingFailed
|
|
225
|
+
case encodingFailed(OSStatus)
|
|
226
|
+
case missingSampleBuffer
|
|
223
227
|
case invalidSampleBuffer
|
|
224
228
|
}
|
|
225
229
|
}
|
|
@@ -39,6 +39,7 @@ actor HIDInjector {
|
|
|
39
39
|
private var hidClient: NSObject?
|
|
40
40
|
private var sendSel: Selector?
|
|
41
41
|
private var simDevice: NSObject?
|
|
42
|
+
private var deviceHubKeyboardBridge: DeviceHubKeyboardBridge?
|
|
42
43
|
|
|
43
44
|
// IndigoHIDMessageForMouseNSEvent(CGPoint*, CGPoint*, IndigoHIDTarget, NSEventType, NSSize, IndigoHIDEdge)
|
|
44
45
|
// arm64 ABI: pointer/int params → x0-x4, float params → d0-d1 (independent numbering).
|
|
@@ -140,6 +141,14 @@ actor HIDInjector {
|
|
|
140
141
|
|
|
141
142
|
self.hidClient = clientObj
|
|
142
143
|
self.sendSel = NSSelectorFromString("sendWithMessage:freeWhenDone:completionQueue:completion:")
|
|
144
|
+
let deviceName = device.value(forKey: "name") as? String ?? deviceUDID
|
|
145
|
+
let runtimeName = (device.value(forKey: "runtime") as? NSObject)?
|
|
146
|
+
.value(forKey: "name") as? String
|
|
147
|
+
self.deviceHubKeyboardBridge = DeviceHubKeyboardBridge.makeIfSupported(
|
|
148
|
+
deviceUDID: deviceUDID,
|
|
149
|
+
deviceName: deviceName,
|
|
150
|
+
runtimeName: runtimeName
|
|
151
|
+
)
|
|
143
152
|
hidLog("[hid] SimDeviceLegacyHIDClient created")
|
|
144
153
|
hidLog("[hid] IndigoHIDMessageForMouseNSEvent loaded (with edge gesture support)")
|
|
145
154
|
}
|
|
@@ -253,11 +262,6 @@ actor HIDInjector {
|
|
|
253
262
|
/// - type: "down" or "up"
|
|
254
263
|
/// - usage: HID usage code (e.g. 0x04 = 'A', 0x28 = Enter, 0xE1 = LeftShift)
|
|
255
264
|
func sendKey(type: String, usage: UInt32) {
|
|
256
|
-
guard let keyboardFunc = keyboardFunc else {
|
|
257
|
-
print("[hid] Keyboard injection unavailable")
|
|
258
|
-
return
|
|
259
|
-
}
|
|
260
|
-
|
|
261
265
|
let direction: UInt32
|
|
262
266
|
switch type {
|
|
263
267
|
case "down": direction = 1
|
|
@@ -265,6 +269,16 @@ actor HIDInjector {
|
|
|
265
269
|
default: return
|
|
266
270
|
}
|
|
267
271
|
|
|
272
|
+
if deviceHubKeyboardBridge?.send(type: type, usage: usage) == true {
|
|
273
|
+
hidLog("[hid] Posted Device Hub key \(type) usage=0x\(String(usage, radix: 16))")
|
|
274
|
+
return
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
guard let keyboardFunc = keyboardFunc else {
|
|
278
|
+
print("[hid] Keyboard injection unavailable")
|
|
279
|
+
return
|
|
280
|
+
}
|
|
281
|
+
|
|
268
282
|
guard let msg = keyboardFunc(usage, direction) else {
|
|
269
283
|
print("[hid] IndigoHIDMessageForKeyboardArbitrary returned nil (usage=0x\(String(usage, radix: 16)))")
|
|
270
284
|
return
|