serve-sim 0.1.45 → 0.1.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/Package.swift CHANGED
@@ -30,9 +30,13 @@ let package = Package(
30
30
  .package(path: "node_modules/node-swift"),
31
31
  ],
32
32
  targets: [
33
+ .target(
34
+ name: "SimNativeSupport"
35
+ ),
33
36
  .target(
34
37
  name: "SimNative",
35
38
  dependencies: [
39
+ "SimNativeSupport",
36
40
  .product(name: "NodeAPI", package: "node-swift"),
37
41
  .product(name: "NodeModuleSupport", package: "node-swift"),
38
42
  ],
@@ -46,6 +50,10 @@ let package = Package(
46
50
  ])
47
51
  ]
48
52
  ),
53
+ .testTarget(
54
+ name: "SimNativeSupportTests",
55
+ dependencies: ["SimNativeSupport"]
56
+ ),
49
57
  ],
50
58
  // The reused SimStreamHelper logic was written against the standalone helper
51
59
  // (plain swiftc, which defaults to Swift 5 mode). Build in Swift 5 mode so its
package/README.md CHANGED
@@ -36,6 +36,8 @@ Requires macOS with Xcode command line tools (`xcrun simctl`) and a [maintained
36
36
 
37
37
  > **Note:** Apple Silicon (arm64) only. The bundled `serve-sim-bin` helper ships as an arm64 binary and does not run on Intel (x86_64) Macs.
38
38
 
39
+ > **Xcode 27 keyboard input:** Device Hub must be running with the target simulator window visible and frontmost. macOS may also require the app that launched `serve-sim` (for example Terminal) to be enabled in **System Settings → Privacy & Security → Accessibility**. Xcode 26 and older keep using the legacy simulator HID path. Set `SERVE_SIM_DISABLE_DEVICE_HUB_KEYBOARD=1` to opt out of the Xcode 27 bridge.
40
+
39
41
  ## CLI
40
42
 
41
43
  ```
@@ -48,7 +48,7 @@ actor CaptureConsumer<E: FrameEncoder>: CaptureConsuming {
48
48
  let encoded = try await encoder.encode(frame)
49
49
  await onFrame(encoded)
50
50
  } catch {
51
- print("error encoding frame: \(error)")
51
+ print("error encoding \(E.self) frame: \(error)")
52
52
  continue
53
53
  }
54
54
  }
@@ -94,8 +94,25 @@ actor CaptureEngine {
94
94
  // drop old frames if there's backpressure
95
95
  bufferingPolicy: .bufferingNewest(1)
96
96
  )
97
- try await frameCapture.start(deviceUDID: deviceUDID) { pixelBuffer, _ in
98
- frameContinuation.yield(Frame(pixelBuffer: pixelBuffer))
97
+ do {
98
+ try await frameCapture.start(deviceUDID: deviceUDID) { pixelBuffer, _ in
99
+ frameContinuation.yield(Frame(pixelBuffer: pixelBuffer))
100
+ }
101
+ } catch {
102
+ // Preserve retryability after an ordinary start failure, but never
103
+ // overwrite a concurrent stop() that moved the actor to .stopped.
104
+ frameContinuation.finish()
105
+ await frameCapture.stop()
106
+ if phase == .starting { phase = .unstarted }
107
+ throw error
108
+ }
109
+ // Actor methods are reentrant across the await above. A simulator can
110
+ // be shut down while FrameCapture is starting; do not resurrect a
111
+ // session that stop() already marked stopped.
112
+ guard phase == .starting else {
113
+ frameContinuation.finish()
114
+ await frameCapture.stop()
115
+ throw CancellationError()
99
116
  }
100
117
  Task {
101
118
  for await frame in frames {
@@ -148,7 +165,7 @@ actor CaptureEngine {
148
165
  let flagDescription: Int32 = 1 << 0
149
166
  let flagKeyframe: Int32 = 1 << 1
150
167
 
151
- guard let self else { return }
168
+ guard let self, let encoded else { return }
152
169
  if let description = encoded.description {
153
170
  await onFrame(
154
171
  screenSize,
@@ -203,13 +220,13 @@ actor AVCCEncoder: FrameEncoder {
203
220
 
204
221
  init() {}
205
222
 
206
- func encode(_ frame: Frame) async throws -> H264Encoder.Encoded {
223
+ func encode(_ frame: Frame) async throws -> H264Encoder.Encoded? {
207
224
  // TODO: cancel after timeout using TaskGroup
208
225
  let result = try await h264Encoder.encode(
209
226
  frame.pixelBuffer,
210
227
  forceKeyframe: forceKeyframe,
211
228
  )
212
- forceKeyframe = false
229
+ if result != nil { forceKeyframe = false }
213
230
  return result
214
231
  }
215
232
 
@@ -0,0 +1,300 @@
1
+ import ApplicationServices
2
+ import AppKit
3
+ import CoreGraphics
4
+ import Foundation
5
+ import SimNativeSupport
6
+
7
+ /// Keyboard transport used by Xcode 27's Device Hub.
8
+ ///
9
+ /// CoreSimulator still accepts legacy Indigo keyboard messages on Xcode 27,
10
+ /// but the iOS 27 guest no longer consumes them. Device Hub owns the new input
11
+ /// route, so this bridge posts ordinary macOS key events to its process and lets
12
+ /// the selected simulator window forward them. Xcode 26 and older never create
13
+ /// the bridge and continue using Indigo unchanged.
14
+ ///
15
+ /// CGEvent can target an application, not one of its windows. To avoid typing
16
+ /// into a different simulator, the first key-down is accepted only when the
17
+ /// target device name identifies one visible, focused Device Hub window through
18
+ /// Accessibility. Accessibility titles remain available when CGWindow titles
19
+ /// are redacted without Screen Recording permission. The route remains latched
20
+ /// until every posted key is released, so one chord never splits between Device
21
+ /// Hub and the legacy Indigo fallback.
22
+ final class DeviceHubKeyboardBridge {
23
+ private static let bundleIdentifier = "com.apple.dt.Devices"
24
+ private static let minimumXcodeMajorVersion = 27
25
+
26
+ private struct ActiveTarget {
27
+ let processIdentifier: pid_t
28
+ }
29
+
30
+ private let expectedBundleURL: URL
31
+ private let deviceUDID: String
32
+ private let deviceName: String
33
+ private let targetWindowTitle: String
34
+
35
+ private var activeTarget: ActiveTarget?
36
+ private var pressedUsages = Set<UInt32>()
37
+ private var lastUnavailableReason: String?
38
+
39
+ private init(
40
+ expectedBundleURL: URL,
41
+ deviceUDID: String,
42
+ deviceName: String,
43
+ runtimeName: String
44
+ ) {
45
+ self.expectedBundleURL = expectedBundleURL.resolvingSymlinksInPath().standardizedFileURL
46
+ self.deviceUDID = deviceUDID
47
+ self.deviceName = deviceName
48
+ self.targetWindowTitle = "\(deviceName) – \(runtimeName)"
49
+ }
50
+
51
+ static func makeIfSupported(
52
+ deviceUDID: String,
53
+ deviceName: String,
54
+ runtimeName: String?
55
+ ) -> DeviceHubKeyboardBridge? {
56
+ let environment = ProcessInfo.processInfo.environment
57
+ guard !DeviceHubKeyboardConfiguration.isDisabled(
58
+ environmentValue: environment["SERVE_SIM_DISABLE_DEVICE_HUB_KEYBOARD"]
59
+ ) else {
60
+ return nil
61
+ }
62
+
63
+ let developerURL = URL(fileURLWithPath: Xcode.developerDir(), isDirectory: true)
64
+ let xcodeURL = developerURL
65
+ .deletingLastPathComponent() // Contents
66
+ .deletingLastPathComponent() // Xcode.app
67
+
68
+ guard let bundle = Bundle(url: xcodeURL), xcodeMajorVersion(in: bundle) >= minimumXcodeMajorVersion else {
69
+ return nil
70
+ }
71
+
72
+ let deviceHubURL = xcodeURL
73
+ .appendingPathComponent("Contents", isDirectory: true)
74
+ .appendingPathComponent("Applications", isDirectory: true)
75
+ .appendingPathComponent("DeviceHub.app", isDirectory: true)
76
+ guard FileManager.default.fileExists(atPath: deviceHubURL.path) else { return nil }
77
+
78
+ guard let runtimeName, !runtimeName.isEmpty else { return nil }
79
+ return DeviceHubKeyboardBridge(
80
+ expectedBundleURL: deviceHubURL,
81
+ deviceUDID: deviceUDID,
82
+ deviceName: deviceName,
83
+ runtimeName: runtimeName
84
+ )
85
+ }
86
+
87
+ /// Returns true only when this event was posted through the guarded Device
88
+ /// Hub route. False lets HIDInjector use Indigo for backward compatibility
89
+ /// and for usages Device Hub cannot safely route.
90
+ func send(type: String, usage: UInt32) -> Bool {
91
+ let keyDown: Bool
92
+ switch type {
93
+ case "down": keyDown = true
94
+ case "up": keyDown = false
95
+ default: return false
96
+ }
97
+
98
+ guard let rawKeyCode = HIDKeyboardMapping.macVirtualKeyCode(for: usage) else { return false }
99
+ guard CGPreflightPostEventAccess() else {
100
+ return unavailable(
101
+ "macOS Accessibility permission is not granted "
102
+ + "(System Settings > Privacy & Security > Accessibility)"
103
+ )
104
+ }
105
+ let keyCode = CGKeyCode(rawKeyCode)
106
+
107
+ let target: ActiveTarget
108
+ if keyDown {
109
+ if let activeTarget {
110
+ guard isExpectedDeviceHubRunning(processIdentifier: activeTarget.processIdentifier) else {
111
+ resetSequence()
112
+ return unavailable("Device Hub exited during a key sequence")
113
+ }
114
+ target = activeTarget
115
+ } else {
116
+ guard let resolved = resolveTarget() else { return false }
117
+ activeTarget = resolved
118
+ target = resolved
119
+ }
120
+ } else {
121
+ // An up event whose down used Indigo must stay on Indigo too.
122
+ guard pressedUsages.contains(usage), let activeTarget else { return false }
123
+ guard isExpectedDeviceHubRunning(processIdentifier: activeTarget.processIdentifier) else {
124
+ resetSequence()
125
+ return unavailable("Device Hub exited during a key sequence")
126
+ }
127
+ target = activeTarget
128
+ }
129
+
130
+ var nextPressedUsages = pressedUsages
131
+ if keyDown {
132
+ nextPressedUsages.insert(usage)
133
+ } else {
134
+ nextPressedUsages.remove(usage)
135
+ }
136
+
137
+ guard
138
+ let source = CGEventSource(stateID: .hidSystemState),
139
+ let event = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: keyDown)
140
+ else {
141
+ if pressedUsages.isEmpty { resetSequence() }
142
+ return unavailable("CoreGraphics could not create a keyboard event")
143
+ }
144
+
145
+ event.flags = Self.eventFlags(for: nextPressedUsages)
146
+ event.postToPid(target.processIdentifier)
147
+
148
+ pressedUsages = nextPressedUsages
149
+ lastUnavailableReason = nil
150
+ if pressedUsages.isEmpty { resetSequence() }
151
+ return true
152
+ }
153
+
154
+ private func resolveTarget() -> ActiveTarget? {
155
+ let applications = runningDeviceHubs()
156
+ guard applications.count == 1, let application = applications.first else {
157
+ let reason = applications.isEmpty
158
+ ? "Device Hub from the selected Xcode is not running"
159
+ : "multiple matching Device Hub processes are running"
160
+ _ = unavailable(reason)
161
+ return nil
162
+ }
163
+
164
+ let processIdentifier = application.processIdentifier
165
+ let route = DeviceHubWindowRouter.route(
166
+ windows: Self.visibleWindows(processIdentifier: processIdentifier),
167
+ processIdentifier: processIdentifier,
168
+ targetWindowTitle: targetWindowTitle
169
+ )
170
+ switch route {
171
+ case .success:
172
+ return ActiveTarget(processIdentifier: processIdentifier)
173
+ case .failure(let failure):
174
+ _ = unavailable(failure.description)
175
+ return nil
176
+ }
177
+ }
178
+
179
+ private func runningDeviceHubs() -> [NSRunningApplication] {
180
+ NSRunningApplication
181
+ .runningApplications(withBundleIdentifier: Self.bundleIdentifier)
182
+ .filter { application in
183
+ guard !application.isTerminated, let bundleURL = application.bundleURL else { return false }
184
+ return bundleURL.resolvingSymlinksInPath().standardizedFileURL == expectedBundleURL
185
+ }
186
+ }
187
+
188
+ private func isExpectedDeviceHubRunning(processIdentifier: pid_t) -> Bool {
189
+ runningDeviceHubs().contains { $0.processIdentifier == processIdentifier }
190
+ }
191
+
192
+ /// Return visible standard Device Hub windows with its key window first.
193
+ /// AX is already covered by the event-posting Accessibility permission and,
194
+ /// unlike CGWindowList, does not require Screen Recording to expose titles.
195
+ private static func visibleWindows(processIdentifier: pid_t) -> [DeviceHubWindow] {
196
+ let application = AXUIElementCreateApplication(processIdentifier)
197
+ guard
198
+ let windows: [AXUIElement] = accessibilityValue(
199
+ kAXWindowsAttribute,
200
+ from: application
201
+ ),
202
+ let focusedWindow: AXUIElement = accessibilityValue(
203
+ kAXFocusedWindowAttribute,
204
+ from: application
205
+ ),
206
+ windows.contains(where: { CFEqual($0, focusedWindow) }),
207
+ isVisibleStandardWindow(focusedWindow) == true
208
+ else {
209
+ return []
210
+ }
211
+
212
+ let orderedWindows = [focusedWindow] + windows.filter { !CFEqual($0, focusedWindow) }
213
+ var snapshot = [DeviceHubWindow]()
214
+ for (index, window) in orderedWindows.enumerated() {
215
+ // A partial AX snapshot is unsafe: dropping an unreadable focused
216
+ // window would make a different window appear to own keyboard focus.
217
+ guard
218
+ let isVisible = isVisibleStandardWindow(window),
219
+ let title: String = accessibilityValue(kAXTitleAttribute, from: window)
220
+ else { return [] }
221
+ guard isVisible else { continue }
222
+ snapshot.append(DeviceHubWindow(
223
+ processIdentifier: processIdentifier,
224
+ // The router only needs stable identity within this snapshot.
225
+ windowNumber: UInt32(index + 1),
226
+ name: title
227
+ ))
228
+ }
229
+ return snapshot
230
+ }
231
+
232
+ private static func isVisibleStandardWindow(_ window: AXUIElement) -> Bool? {
233
+ guard
234
+ let role: String = accessibilityValue(kAXRoleAttribute, from: window),
235
+ let subrole: String = accessibilityValue(kAXSubroleAttribute, from: window),
236
+ let minimized: Bool = accessibilityValue(kAXMinimizedAttribute, from: window)
237
+ else { return nil }
238
+ return role == kAXWindowRole && subrole == kAXStandardWindowSubrole && !minimized
239
+ }
240
+
241
+ private static func accessibilityValue<T>(
242
+ _ attribute: String,
243
+ from element: AXUIElement
244
+ ) -> T? {
245
+ var value: CFTypeRef?
246
+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {
247
+ return nil
248
+ }
249
+ return value as? T
250
+ }
251
+
252
+ private func unavailable(_ reason: String) -> Bool {
253
+ if lastUnavailableReason != reason {
254
+ let shortUDID = String(deviceUDID.prefix(8))
255
+ print(
256
+ "[hid] Device Hub keyboard unavailable for \(deviceName) (\(shortUDID)): "
257
+ + "\(reason); using legacy HID"
258
+ )
259
+ lastUnavailableReason = reason
260
+ }
261
+ return false
262
+ }
263
+
264
+ private func resetSequence() {
265
+ activeTarget = nil
266
+ pressedUsages.removeAll()
267
+ }
268
+
269
+ private static func eventFlags(for usages: Set<UInt32>) -> CGEventFlags {
270
+ var flags: CGEventFlags = []
271
+ for usage in usages {
272
+ switch HIDKeyboardMapping.modifier(for: usage) {
273
+ case .control: flags.insert(.maskControl)
274
+ case .shift: flags.insert(.maskShift)
275
+ case .option: flags.insert(.maskAlternate)
276
+ case .command: flags.insert(.maskCommand)
277
+ case nil: break
278
+ }
279
+ }
280
+ return flags
281
+ }
282
+
283
+ private static func xcodeMajorVersion(in bundle: Bundle) -> Int {
284
+ if
285
+ let version = bundle.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String,
286
+ let major = Int(version.split(separator: ".").first ?? "")
287
+ {
288
+ return major
289
+ }
290
+
291
+ if let dtxcode = bundle.object(forInfoDictionaryKey: "DTXcode") as? String,
292
+ let value = Int(dtxcode) {
293
+ return value / 100
294
+ }
295
+ if let dtxcode = bundle.object(forInfoDictionaryKey: "DTXcode") as? NSNumber {
296
+ return dtxcode.intValue / 100
297
+ }
298
+ return 0
299
+ }
300
+ }
@@ -4,6 +4,7 @@ import CoreMedia
4
4
  import CoreGraphics
5
5
  import IOSurface
6
6
  import ObjectiveC
7
+ import SimNativeSupport
7
8
 
8
9
  /// Headless simulator frame capture via direct IOSurface access.
9
10
  ///
@@ -40,6 +41,9 @@ actor FrameCapture {
40
41
  private var descriptors: [NSObject] = []
41
42
  private var callbackUUIDs: [ObjectIdentifier: UUID] = [:]
42
43
  private var ioClient: NSObject?
44
+ private var expectedScreenSize: FramebufferSurfaceSize?
45
+ private var didLogRejectedPresentationSurface = false
46
+ private var didLogMissingExpectedSurface = false
43
47
 
44
48
  func start(deviceUDID: String, onFrame: @escaping @Sendable (CVPixelBuffer, CMTime) -> Void) throws {
45
49
  self.onFrame = onFrame
@@ -53,6 +57,7 @@ actor FrameCapture {
53
57
  guard state == "Booted" else {
54
58
  throw makeError(2, "Device not booted (state: \(state))")
55
59
  }
60
+ self.expectedScreenSize = Self.nativeScreenSize(for: device)
56
61
 
57
62
  guard let io = device.perform(NSSelectorFromString("io"))?.takeUnretainedValue() as? NSObject else {
58
63
  throw makeError(3, "Failed to get device IO")
@@ -139,22 +144,56 @@ actor FrameCapture {
139
144
  return candidates
140
145
  }
141
146
 
142
- /// Return the descriptor whose live surface has the largest area.
143
- /// Secondary planes/overlays are typically smaller than the main screen.
147
+ /// Return the descriptor matching the device type's native screen size.
148
+ /// Xcode 27 can expose an additional 7680x4320 presentation surface while
149
+ /// Device Hub is resizing; selecting the historical largest surface then
150
+ /// feeds a non-device frame to VideoToolbox and produces `encodingFailed`.
151
+ /// If screen metadata is unavailable, retain the old largest-live fallback.
144
152
  private func pickBestDescriptor() -> NSObject? {
145
153
  let surfSel = NSSelectorFromString("framebufferSurface")
146
- var best: NSObject?
147
- var bestArea: Int = 0
148
- for desc in descriptors {
149
- guard let surfObj = desc.perform(surfSel)?.takeUnretainedValue() else { continue }
154
+ let sizes = descriptors.map { desc -> FramebufferSurfaceSize in
155
+ guard let surfObj = desc.perform(surfSel)?.takeUnretainedValue() else {
156
+ return FramebufferSurfaceSize(width: 0, height: 0)
157
+ }
150
158
  let surf = unsafeBitCast(surfObj, to: IOSurface.self)
151
- let area = IOSurfaceGetWidth(surf) * IOSurfaceGetHeight(surf)
152
- if area > bestArea {
153
- best = desc
154
- bestArea = area
159
+ return FramebufferSurfaceSize(
160
+ width: IOSurfaceGetWidth(surf),
161
+ height: IOSurfaceGetHeight(surf)
162
+ )
163
+ }
164
+ guard let selection = FramebufferSurfaceSelector.select(
165
+ from: sizes,
166
+ expectedSize: expectedScreenSize
167
+ ) else {
168
+ return nil
169
+ }
170
+
171
+ if selection.matchedExpectedSize, !didLogRejectedPresentationSurface {
172
+ let selected = sizes[selection.index]
173
+ let selectedArea = Int64(selected.width) * Int64(selected.height)
174
+ if let larger = sizes.filter(\.isLive).max(by: {
175
+ Int64($0.width) * Int64($0.height) < Int64($1.width) * Int64($1.height)
176
+ }), Int64(larger.width) * Int64(larger.height) > selectedArea {
177
+ print(
178
+ "[capture] Ignoring non-device framebuffer \(larger.width)x\(larger.height); "
179
+ + "native screen is \(selected.width)x\(selected.height)"
180
+ )
181
+ didLogRejectedPresentationSurface = true
155
182
  }
183
+ } else if
184
+ !selection.matchedExpectedSize,
185
+ let expectedScreenSize,
186
+ !didLogMissingExpectedSurface
187
+ {
188
+ let selected = sizes[selection.index]
189
+ print(
190
+ "[capture] No framebuffer matches native screen "
191
+ + "\(expectedScreenSize.width)x\(expectedScreenSize.height); "
192
+ + "falling back to \(selected.width)x\(selected.height)"
193
+ )
194
+ didLogMissingExpectedSurface = true
156
195
  }
157
- return best
196
+ return descriptors[selection.index]
158
197
  }
159
198
 
160
199
  // MARK: - Frame callbacks via objc_msgSend
@@ -270,7 +309,16 @@ actor FrameCapture {
270
309
  callbackUUIDs.removeAll()
271
310
  descriptors.removeAll()
272
311
  lastSeeds.removeAll()
312
+ onFrame = nil
313
+ frameCount = 0
314
+ capturedWidth = 0
315
+ capturedHeight = 0
316
+ rewireTickCount = 0
317
+ lastCaptureTime = .now
273
318
  ioClient = nil
319
+ expectedScreenSize = nil
320
+ didLogRejectedPresentationSurface = false
321
+ didLogMissingExpectedSurface = false
274
322
  }
275
323
 
276
324
  // MARK: - Helpers
@@ -280,6 +328,34 @@ actor FrameCapture {
280
328
  userInfo: [NSLocalizedDescriptionKey: msg])
281
329
  }
282
330
 
331
+ /// SimDeviceType.mainScreenSize is private but stable across the same
332
+ /// SimulatorKit versions already used by this file. Runtime validation on
333
+ /// Xcode 27 confirms that it reports native pixels, not logical points, and
334
+ /// matches the primary IOSurface exactly. The `@convention(c)` IMP type is
335
+ /// intentional: it preserves the platform CGSize return ABI (registers on
336
+ /// arm64 and the appropriate struct-return convention on x86_64).
337
+ private static func nativeScreenSize(for device: NSObject) -> FramebufferSurfaceSize? {
338
+ let deviceTypeSelector = NSSelectorFromString("deviceType")
339
+ guard
340
+ device.responds(to: deviceTypeSelector),
341
+ let deviceType = device.perform(deviceTypeSelector)?.takeUnretainedValue() as? NSObject
342
+ else {
343
+ return nil
344
+ }
345
+ let selector = NSSelectorFromString("mainScreenSize")
346
+ guard deviceType.responds(to: selector) else { return nil }
347
+
348
+ typealias GetSize = @convention(c) (AnyObject, Selector) -> CGSize
349
+ let getSize = unsafeBitCast(deviceType.method(for: selector), to: GetSize.self)
350
+ let size = getSize(deviceType, selector)
351
+ guard size.width.isFinite, size.height.isFinite else { return nil }
352
+
353
+ let width = Int(size.width.rounded())
354
+ let height = Int(size.height.rounded())
355
+ guard width > 0, height > 0 else { return nil }
356
+ return FramebufferSurfaceSize(width: width, height: height)
357
+ }
358
+
283
359
  static func findSimDevice(udid: String) -> NSObject? {
284
360
  guard let contextClass = NSClassFromString("SimServiceContext") as? NSObject.Type else { return nil }
285
361
  let developerDir = Xcode.developerDir()
@@ -41,8 +41,9 @@ actor H264Encoder {
41
41
  if let session { VTCompressionSessionInvalidate(session) }
42
42
  }
43
43
 
44
- /// Submit a frame. Returns immediately; `onEncoded` fires on VT's queue.
45
- func encode(_ source: CVPixelBuffer, forceKeyframe: Bool = false) async throws -> Encoded {
44
+ /// Encode one frame. Real-time VideoToolbox sessions may deliberately drop
45
+ /// a frame under pressure; that is reported as `nil`, not as an error.
46
+ func encode(_ source: CVPixelBuffer, forceKeyframe: Bool = false) async throws -> Encoded? {
46
47
  let w = Int32(CVPixelBufferGetWidth(source))
47
48
  let h = Int32(CVPixelBufferGetHeight(source))
48
49
  if session == nil || w != width || h != height {
@@ -60,7 +61,7 @@ actor H264Encoder {
60
61
  ? [kVTEncodeFrameOptionKey_ForceKeyFrame: kCFBooleanTrue!] as NSDictionary
61
62
  : nil
62
63
 
63
- let buffer: CMSampleBuffer? = await withCheckedContinuation { continuation in
64
+ let result: (buffer: CMSampleBuffer?, status: OSStatus, flags: VTEncodeInfoFlags) = await withCheckedContinuation { continuation in
64
65
  let status = VTCompressionSessionEncodeFrame(
65
66
  session,
66
67
  imageBuffer: source,
@@ -68,18 +69,20 @@ actor H264Encoder {
68
69
  duration: .invalid,
69
70
  frameProperties: frameProps,
70
71
  infoFlagsOut: nil
71
- ) { @Sendable status, _, sampleBuffer in
72
- guard status == noErr, let sb = sampleBuffer else {
73
- continuation.resume(returning: nil)
72
+ ) { @Sendable status, flags, sampleBuffer in
73
+ guard status == noErr, let sampleBuffer else {
74
+ continuation.resume(returning: (nil, status, flags))
74
75
  return
75
76
  }
76
- continuation.resume(returning: sb)
77
+ continuation.resume(returning: (sampleBuffer, status, flags))
77
78
  }
78
79
  if status != noErr {
79
- continuation.resume(returning: nil)
80
+ continuation.resume(returning: (nil, status, []))
80
81
  }
81
82
  }
82
- guard let buffer else { throw Errors.encodingFailed }
83
+ if result.status == noErr, result.flags.contains(.frameDropped) { return nil }
84
+ guard result.status == noErr else { throw Errors.encodingFailed(result.status) }
85
+ guard let buffer = result.buffer else { throw Errors.missingSampleBuffer }
83
86
  return try extract(from: buffer)
84
87
  }
85
88
 
@@ -219,7 +222,8 @@ actor H264Encoder {
219
222
 
220
223
  enum Errors: Error {
221
224
  case couldNotCreateSession
222
- case encodingFailed
225
+ case encodingFailed(OSStatus)
226
+ case missingSampleBuffer
223
227
  case invalidSampleBuffer
224
228
  }
225
229
  }
@@ -39,6 +39,7 @@ actor HIDInjector {
39
39
  private var hidClient: NSObject?
40
40
  private var sendSel: Selector?
41
41
  private var simDevice: NSObject?
42
+ private var deviceHubKeyboardBridge: DeviceHubKeyboardBridge?
42
43
 
43
44
  // IndigoHIDMessageForMouseNSEvent(CGPoint*, CGPoint*, IndigoHIDTarget, NSEventType, NSSize, IndigoHIDEdge)
44
45
  // arm64 ABI: pointer/int params → x0-x4, float params → d0-d1 (independent numbering).
@@ -140,6 +141,14 @@ actor HIDInjector {
140
141
 
141
142
  self.hidClient = clientObj
142
143
  self.sendSel = NSSelectorFromString("sendWithMessage:freeWhenDone:completionQueue:completion:")
144
+ let deviceName = device.value(forKey: "name") as? String ?? deviceUDID
145
+ let runtimeName = (device.value(forKey: "runtime") as? NSObject)?
146
+ .value(forKey: "name") as? String
147
+ self.deviceHubKeyboardBridge = DeviceHubKeyboardBridge.makeIfSupported(
148
+ deviceUDID: deviceUDID,
149
+ deviceName: deviceName,
150
+ runtimeName: runtimeName
151
+ )
143
152
  hidLog("[hid] SimDeviceLegacyHIDClient created")
144
153
  hidLog("[hid] IndigoHIDMessageForMouseNSEvent loaded (with edge gesture support)")
145
154
  }
@@ -253,11 +262,6 @@ actor HIDInjector {
253
262
  /// - type: "down" or "up"
254
263
  /// - usage: HID usage code (e.g. 0x04 = 'A', 0x28 = Enter, 0xE1 = LeftShift)
255
264
  func sendKey(type: String, usage: UInt32) {
256
- guard let keyboardFunc = keyboardFunc else {
257
- print("[hid] Keyboard injection unavailable")
258
- return
259
- }
260
-
261
265
  let direction: UInt32
262
266
  switch type {
263
267
  case "down": direction = 1
@@ -265,6 +269,16 @@ actor HIDInjector {
265
269
  default: return
266
270
  }
267
271
 
272
+ if deviceHubKeyboardBridge?.send(type: type, usage: usage) == true {
273
+ hidLog("[hid] Posted Device Hub key \(type) usage=0x\(String(usage, radix: 16))")
274
+ return
275
+ }
276
+
277
+ guard let keyboardFunc = keyboardFunc else {
278
+ print("[hid] Keyboard injection unavailable")
279
+ return
280
+ }
281
+
268
282
  guard let msg = keyboardFunc(usage, direction) else {
269
283
  print("[hid] IndigoHIDMessageForKeyboardArbitrary returned nil (usage=0x\(String(usage, radix: 16)))")
270
284
  return