serve-sim 0.1.44 → 0.1.45-beta.116.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/Sources/SimCameraHelper/main.m +3 -1
- package/Sources/SimNative/CaptureEngine.swift +219 -0
- package/Sources/SimNative/FrameCapture.swift +58 -60
- package/Sources/SimNative/H264Encoder.swift +43 -79
- package/Sources/SimNative/HIDInjector.swift +80 -93
- package/Sources/SimNative/PixelBufferUtils.swift +62 -0
- package/Sources/SimNative/VideoEncoder.swift +23 -16
- package/Sources/SimNative/sim-module.swift +90 -60
- package/dist/middleware.js +48 -35
- package/dist/native/serve-sim-native.node +0 -0
- package/dist/serve-sim.js +89 -76
- package/dist/simcam/serve-sim-camera-helper +0 -0
- package/package.json +2 -1
- package/src/camera-helper.ts +150 -0
- package/src/middleware.ts +376 -144
- package/src/native.ts +71 -71
- package/Sources/SimNative/sim-capture.swift +0 -224
package/README.md
CHANGED
|
@@ -19,6 +19,7 @@ https://github.com/user-attachments/assets/fbf890f4-c8c7-4684-82be-d677b8a188f8
|
|
|
19
19
|
- Swipe from the bottom to go home.
|
|
20
20
|
- gestures like pinch to zoom by holding the option key.
|
|
21
21
|
- Simulator logs are forwarded to the browser for browser-use MCP tools to read from.
|
|
22
|
+
- Recent simulator actions are available in the browser tools panel and `serve-sim event-log`.
|
|
22
23
|
- Drag and drop videos and images to add them to the simulator device.
|
|
23
24
|
- Keyboard commands and hot keys are forwarded to the simulator, including CMD+SHIFT+H to go home.
|
|
24
25
|
- Apple Watch, iPad, and iOS support.
|
|
@@ -51,6 +52,7 @@ serve-sim ca-debug <option> <on|off> [-d udid]
|
|
|
51
52
|
Toggle a CoreAnimation debug flag
|
|
52
53
|
(blended|copies|misaligned|offscreen|slow-animations)
|
|
53
54
|
serve-sim memory-warning [-d udid] Simulate a memory warning
|
|
55
|
+
serve-sim event-log [-d udid] Show recent simulator events
|
|
54
56
|
|
|
55
57
|
serve-sim camera <bundle-id> [-d udid] [source-options]
|
|
56
58
|
Inject a synthetic camera feed and (re)launch the app
|
|
@@ -68,6 +70,11 @@ Options:
|
|
|
68
70
|
-d, --detach Spawn helper and exit (daemon mode)
|
|
69
71
|
-q, --quiet JSON-only output
|
|
70
72
|
--no-preview Skip the web UI; stream in foreground only
|
|
73
|
+
--panes <panes> Initially open preview panes: devices, tools, devtools,
|
|
74
|
+
or none
|
|
75
|
+
--fit Initially size the simulator to fit the preview viewport
|
|
76
|
+
--theme <theme> Set simulator appearance before opening the preview:
|
|
77
|
+
light or dark
|
|
71
78
|
--codec <codec> Stream codec for the preview UI: 'auto' (H.264 when the
|
|
72
79
|
browser can decode it) or 'mjpeg' (force software JPEG —
|
|
73
80
|
e.g. on VMs without H.264 encode)
|
|
@@ -94,6 +101,8 @@ serve-sim "iPhone 16 Pro" # target a specific device
|
|
|
94
101
|
serve-sim --detach # start a background helper, return JSON
|
|
95
102
|
serve-sim --list # show running streams
|
|
96
103
|
serve-sim --kill # stop all helpers
|
|
104
|
+
serve-sim --panes devices,tools --fit # start with selected panes open and fit the simulator
|
|
105
|
+
serve-sim --theme dark # start the simulator in Dark Mode
|
|
97
106
|
|
|
98
107
|
# Type text into the focused field
|
|
99
108
|
serve-sim type "Hello, world!"
|
|
@@ -931,11 +931,13 @@ int main(int argc, const char *argv[]) {
|
|
|
931
931
|
}
|
|
932
932
|
if (gAcceptSource) dispatch_source_cancel(gAcceptSource);
|
|
933
933
|
if (gControlListenFd >= 0) { close(gControlListenFd); if (socketPath) unlink(socketPath); }
|
|
934
|
+
// Unlink the shm name before stopping capture sources: if a source
|
|
935
|
+
// teardown crashes, the name must not stay resolvable forever.
|
|
936
|
+
if (gShmName) shm_unlink(gShmName);
|
|
934
937
|
StopPlaceholderSource();
|
|
935
938
|
StopWebcamSource();
|
|
936
939
|
StopVideoSource();
|
|
937
940
|
ReleaseSurfaces();
|
|
938
|
-
if (gShmName) shm_unlink(gShmName);
|
|
939
941
|
fprintf(stderr, "[serve-sim-camera] stopped\n");
|
|
940
942
|
return 0;
|
|
941
943
|
}
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import CoreVideo
|
|
3
|
+
import CoreMedia
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
// The capture + encode engine, reused verbatim from SimStreamHelper. Replicates
|
|
7
|
+
// main.swift's frameHandler: MJPEG always encodes while clients exist; H.264 runs
|
|
8
|
+
// only while AVCC is active. Encoded bytes (JPEG, or natively-framed AVCC
|
|
9
|
+
// envelopes) are handed back through a Swift closure on a native encode thread;
|
|
10
|
+
// the node-swift binding (sim-module.swift) marshals them onto the JS thread via
|
|
11
|
+
// a NodeAsyncQueue (threadsafe function).
|
|
12
|
+
|
|
13
|
+
struct Frame: Identifiable {
|
|
14
|
+
let id = UUID()
|
|
15
|
+
let pixelBuffer: CVPixelBuffer
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
protocol FrameEncoder {
|
|
19
|
+
associatedtype Encoded
|
|
20
|
+
func encode(_ frame: Frame) async throws -> Encoded
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
protocol CaptureConsuming: Sendable {
|
|
24
|
+
// this is intentionally synchronous. CaptureEngine sends all frames to all consumers,
|
|
25
|
+
// and lets them handle internal backpressure as they see fit. if instead this were async
|
|
26
|
+
// (and CaptureEngine waited for all consumers to finish), a single bad consumer could
|
|
27
|
+
// jam up the entire pipeline.
|
|
28
|
+
func handleFrame(_ frame: Frame)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
actor CaptureConsumer<E: FrameEncoder>: CaptureConsuming {
|
|
32
|
+
nonisolated let continuation: AsyncStream<Frame>.Continuation
|
|
33
|
+
|
|
34
|
+
init(
|
|
35
|
+
encoder: E,
|
|
36
|
+
onFrame: @escaping @isolated(any) (E.Encoded) async -> Void
|
|
37
|
+
) {
|
|
38
|
+
let (stream, continuation) = AsyncStream.makeStream(
|
|
39
|
+
of: Frame.self,
|
|
40
|
+
// drop old frames if there's backpressure
|
|
41
|
+
bufferingPolicy: .bufferingNewest(1)
|
|
42
|
+
)
|
|
43
|
+
self.continuation = continuation
|
|
44
|
+
Task {
|
|
45
|
+
_ = onFrame.isolation
|
|
46
|
+
for await frame in stream {
|
|
47
|
+
do {
|
|
48
|
+
let encoded = try await encoder.encode(frame)
|
|
49
|
+
await onFrame(encoded)
|
|
50
|
+
} catch {
|
|
51
|
+
print("error encoding frame: \(error)")
|
|
52
|
+
continue
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
nonisolated func handleFrame(_ frame: Frame) {
|
|
59
|
+
continuation.yield(frame)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
deinit { continuation.finish() }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
actor CaptureEngine {
|
|
66
|
+
private enum Phase {
|
|
67
|
+
case unstarted
|
|
68
|
+
case starting
|
|
69
|
+
case running
|
|
70
|
+
case stopped
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private let deviceUDID: String
|
|
74
|
+
private let frameCapture = FrameCapture()
|
|
75
|
+
private var phase = Phase.unstarted
|
|
76
|
+
|
|
77
|
+
// mjpeg is stateless so we can share a single encoder instance
|
|
78
|
+
private let mjpegEncoder = MJPEGEncoder()
|
|
79
|
+
|
|
80
|
+
private(set) var screenSize = Dimensions(width: 0, height: 0)
|
|
81
|
+
private var consumers = [UUID: CaptureConsuming]()
|
|
82
|
+
|
|
83
|
+
init(deviceUDID: String) {
|
|
84
|
+
self.deviceUDID = deviceUDID
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
func start() async throws {
|
|
88
|
+
guard phase == .unstarted else { return }
|
|
89
|
+
phase = .starting
|
|
90
|
+
// Latch `started` only after capture actually begins: if start() throws
|
|
91
|
+
// (e.g. device not booted), a later retry should still be allowed.
|
|
92
|
+
let (frames, frameContinuation) = AsyncStream.makeStream(
|
|
93
|
+
of: Frame.self,
|
|
94
|
+
// drop old frames if there's backpressure
|
|
95
|
+
bufferingPolicy: .bufferingNewest(1)
|
|
96
|
+
)
|
|
97
|
+
try await frameCapture.start(deviceUDID: deviceUDID) { pixelBuffer, _ in
|
|
98
|
+
frameContinuation.yield(Frame(pixelBuffer: pixelBuffer))
|
|
99
|
+
}
|
|
100
|
+
Task {
|
|
101
|
+
for await frame in frames {
|
|
102
|
+
handleFrame(frame)
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
phase = .running
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private func addConsumer<E: FrameEncoder>(
|
|
109
|
+
encoder: E,
|
|
110
|
+
onFrame: sending @escaping @isolated(any) (E.Encoded) async -> Void
|
|
111
|
+
) -> (@Sendable () async -> Void) {
|
|
112
|
+
let consumer = CaptureConsumer(encoder: encoder) { [weak self] encoded in
|
|
113
|
+
guard let self, await self.phase == .running else { return }
|
|
114
|
+
await onFrame(encoded)
|
|
115
|
+
}
|
|
116
|
+
let id = UUID()
|
|
117
|
+
consumers[id] = consumer
|
|
118
|
+
return { await self.removeConsumer(id) }
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
private func removeConsumer(
|
|
122
|
+
_ id: UUID
|
|
123
|
+
) {
|
|
124
|
+
consumers.removeValue(forKey: id)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
private func handleFrame(_ frame: Frame) {
|
|
128
|
+
guard phase == .running else { return }
|
|
129
|
+
screenSize = frame.pixelBuffer.dimensions
|
|
130
|
+
for consumer in consumers.values {
|
|
131
|
+
consumer.handleFrame(frame)
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
func addMJPEGConsumer(
|
|
136
|
+
onFrame: sending @escaping (Dimensions, Data) async -> Void
|
|
137
|
+
) -> (@Sendable () async -> Void) {
|
|
138
|
+
return addConsumer(encoder: mjpegEncoder, onFrame: { [weak self] data in
|
|
139
|
+
guard let self else { return }
|
|
140
|
+
await onFrame(screenSize, data)
|
|
141
|
+
})
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
func addAVCCConsumer(
|
|
145
|
+
onFrame: sending @escaping (Dimensions, Data, Int32) async -> Void
|
|
146
|
+
) -> (@Sendable () async -> Void) {
|
|
147
|
+
addConsumer(encoder: AVCCEncoder()) { [weak self] encoded in
|
|
148
|
+
let flagDescription: Int32 = 1 << 0
|
|
149
|
+
let flagKeyframe: Int32 = 1 << 1
|
|
150
|
+
|
|
151
|
+
guard let self else { return }
|
|
152
|
+
if let description = encoded.description {
|
|
153
|
+
await onFrame(
|
|
154
|
+
screenSize,
|
|
155
|
+
AVCCEnvelope.description(avcc: description),
|
|
156
|
+
flagDescription,
|
|
157
|
+
)
|
|
158
|
+
}
|
|
159
|
+
switch encoded.kind {
|
|
160
|
+
case .keyframe:
|
|
161
|
+
await onFrame(
|
|
162
|
+
screenSize,
|
|
163
|
+
AVCCEnvelope.keyframe(avcc: encoded.avcc),
|
|
164
|
+
flagKeyframe,
|
|
165
|
+
)
|
|
166
|
+
case .delta:
|
|
167
|
+
await onFrame(
|
|
168
|
+
screenSize,
|
|
169
|
+
AVCCEnvelope.delta(avcc: encoded.avcc),
|
|
170
|
+
0,
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
func stop() {
|
|
177
|
+
if phase == .stopped { return }
|
|
178
|
+
phase = .stopped
|
|
179
|
+
Task { [frameCapture] in await frameCapture.stop() }
|
|
180
|
+
consumers.removeAll()
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
actor MJPEGEncoder: FrameEncoder {
|
|
185
|
+
private let videoEncoder = VideoEncoder(quality: 0.7)
|
|
186
|
+
private var lastImage: (UUID, Data)?
|
|
187
|
+
|
|
188
|
+
init() {}
|
|
189
|
+
|
|
190
|
+
func encode(_ frame: Frame) async throws -> Data {
|
|
191
|
+
if let (id, data) = lastImage, id == frame.id { return data }
|
|
192
|
+
let data = try await videoEncoder.encode(pixelBuffer: frame.pixelBuffer)
|
|
193
|
+
lastImage = (frame.id, data)
|
|
194
|
+
return data
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
actor AVCCEncoder: FrameEncoder {
|
|
199
|
+
private static let timeout: Duration = .milliseconds(500)
|
|
200
|
+
|
|
201
|
+
let h264Encoder = H264Encoder(fps: 60)
|
|
202
|
+
var forceKeyframe = true
|
|
203
|
+
|
|
204
|
+
init() {}
|
|
205
|
+
|
|
206
|
+
func encode(_ frame: Frame) async throws -> H264Encoder.Encoded {
|
|
207
|
+
// TODO: cancel after timeout using TaskGroup
|
|
208
|
+
let result = try await h264Encoder.encode(
|
|
209
|
+
frame.pixelBuffer,
|
|
210
|
+
forceKeyframe: forceKeyframe,
|
|
211
|
+
)
|
|
212
|
+
forceKeyframe = false
|
|
213
|
+
return result
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
deinit {
|
|
217
|
+
Task { [h264Encoder] in await h264Encoder.stop() }
|
|
218
|
+
}
|
|
219
|
+
}
|
|
@@ -12,14 +12,17 @@ import ObjectiveC
|
|
|
12
12
|
/// for late-joining clients.
|
|
13
13
|
///
|
|
14
14
|
/// Pipeline: IOSurface (shared memory) → CVPixelBuffer (zero-copy) → H.264 encode
|
|
15
|
-
|
|
15
|
+
actor FrameCapture {
|
|
16
|
+
private let queue = DispatchSerialQueue(label: "frame-capture", qos: .userInteractive)
|
|
17
|
+
nonisolated var unownedExecutor: UnownedSerialExecutor { queue.asUnownedSerialExecutor() }
|
|
18
|
+
|
|
19
|
+
private var photocopier = Photocopier()
|
|
16
20
|
private var onFrame: ((CVPixelBuffer, CMTime) -> Void)?
|
|
17
21
|
private var frameCount: UInt64 = 0
|
|
18
22
|
private(set) var capturedWidth: Int = 0
|
|
19
23
|
private(set) var capturedHeight: Int = 0
|
|
20
|
-
private var idleTimer:
|
|
21
|
-
private
|
|
22
|
-
private var lastCaptureTimeMs: UInt64 = 0
|
|
24
|
+
private var idleTimer: Task<Void, Never>?
|
|
25
|
+
private var lastCaptureTime: ContinuousClock.Instant = .now
|
|
23
26
|
private var lastSeeds: [ObjectIdentifier: UInt32] = [:]
|
|
24
27
|
private var rewireTickCount: Int = 0
|
|
25
28
|
/// Interval at which the idle timer re-emits the current frame even when
|
|
@@ -32,13 +35,13 @@ final class FrameCapture {
|
|
|
32
35
|
/// one subscriber is due for it — a late-joining relay subscriber on an
|
|
33
36
|
/// idle sim never gets a cached frame to show.
|
|
34
37
|
/// Re-emitting at ~5 fps fixes both without meaningful CPU cost.
|
|
35
|
-
private static let
|
|
38
|
+
private static let idleInterval: ContinuousClock.Duration = .milliseconds(200)
|
|
36
39
|
|
|
37
40
|
private var descriptors: [NSObject] = []
|
|
38
|
-
private var callbackUUIDs: [ObjectIdentifier:
|
|
41
|
+
private var callbackUUIDs: [ObjectIdentifier: UUID] = [:]
|
|
39
42
|
private var ioClient: NSObject?
|
|
40
43
|
|
|
41
|
-
func start(deviceUDID: String, onFrame: @escaping (CVPixelBuffer, CMTime) -> Void) throws {
|
|
44
|
+
func start(deviceUDID: String, onFrame: @escaping @Sendable (CVPixelBuffer, CMTime) -> Void) throws {
|
|
42
45
|
self.onFrame = onFrame
|
|
43
46
|
|
|
44
47
|
SimFrameworks.load()
|
|
@@ -156,70 +159,56 @@ final class FrameCapture {
|
|
|
156
159
|
|
|
157
160
|
// MARK: - Frame callbacks via objc_msgSend
|
|
158
161
|
|
|
159
|
-
private func registerFrameCallbacks(desc:
|
|
160
|
-
let regSel =
|
|
162
|
+
private func registerFrameCallbacks(desc: AnyObject) throws {
|
|
163
|
+
let regSel = #selector(FramebufferDescriptor.registerScreenCallbacks)
|
|
161
164
|
guard desc.responds(to: regSel) else {
|
|
162
165
|
throw makeError(8, "Descriptor doesn't support registerScreenCallbacks")
|
|
163
166
|
}
|
|
164
167
|
|
|
165
|
-
|
|
166
|
-
throw makeError(9, "objc_msgSend not found")
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
typealias MsgSendFunc = @convention(c) (
|
|
170
|
-
AnyObject, Selector, AnyObject, AnyObject, AnyObject, AnyObject, AnyObject
|
|
171
|
-
) -> Void
|
|
172
|
-
let msgSend = unsafeBitCast(msgSendPtr, to: MsgSendFunc.self)
|
|
173
|
-
|
|
174
|
-
let uuid = NSUUID()
|
|
168
|
+
let uuid = UUID()
|
|
175
169
|
callbackUUIDs[ObjectIdentifier(desc)] = uuid
|
|
176
170
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
self
|
|
182
|
-
|
|
183
|
-
let propsCallback: @convention(block) () -> Void = {}
|
|
184
|
-
|
|
185
|
-
msgSend(
|
|
186
|
-
desc, regSel,
|
|
187
|
-
uuid, captureQueue as AnyObject,
|
|
188
|
-
frameCallback as AnyObject, surfacesCallback as AnyObject, propsCallback as AnyObject
|
|
171
|
+
desc.registerScreenCallbacks(
|
|
172
|
+
uuid: uuid,
|
|
173
|
+
callbackQueue: queue,
|
|
174
|
+
frameCallback: { [self] in assumeIsolated { $0.captureFrame() } },
|
|
175
|
+
surfacesChangedCallback: { [self] in assumeIsolated { $0.captureFrame() } },
|
|
176
|
+
propertiesChangedCallback: {}
|
|
189
177
|
)
|
|
190
178
|
}
|
|
191
179
|
|
|
192
180
|
private func startIdleTimer() {
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
let nowMs = DispatchTime.now().uptimeNanoseconds / 1_000_000
|
|
199
|
-
if (nowMs - self.lastCaptureTimeMs) >= Self.idleIntervalMs {
|
|
200
|
-
self.captureFrame()
|
|
181
|
+
self.idleTimer = Task { [weak self] in
|
|
182
|
+
while !Task.isCancelled {
|
|
183
|
+
guard let self else { return }
|
|
184
|
+
await self.onIdleTimerTick()
|
|
185
|
+
try? await Task.sleep(for: Self.idleInterval)
|
|
201
186
|
}
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private func onIdleTimerTick() {
|
|
191
|
+
let now = ContinuousClock.now
|
|
192
|
+
guard (now - self.lastCaptureTime) >= Self.idleInterval else { return }
|
|
193
|
+
self.captureFrame(force: true)
|
|
194
|
+
// Self-heal: if we've never captured a frame, the cached descriptor
|
|
195
|
+
// is likely stale. Re-wire the pipeline periodically (every ~1s)
|
|
196
|
+
// until frames start flowing.
|
|
197
|
+
if self.frameCount == 0 {
|
|
198
|
+
self.rewireTickCount += 1
|
|
199
|
+
if self.rewireTickCount % 5 == 0 {
|
|
200
|
+
do {
|
|
201
|
+
try self.wireUpFramebuffer()
|
|
202
|
+
} catch {
|
|
203
|
+
// Swallow — we'll try again on the next tick.
|
|
213
204
|
}
|
|
214
205
|
}
|
|
215
206
|
}
|
|
216
|
-
timer.resume()
|
|
217
|
-
self.idleTimer = timer
|
|
218
207
|
}
|
|
219
208
|
|
|
220
209
|
// MARK: - Frame capture
|
|
221
210
|
|
|
222
|
-
private func captureFrame() {
|
|
211
|
+
private func captureFrame(force: Bool = false) {
|
|
223
212
|
guard let desc = pickBestDescriptor() else { return }
|
|
224
213
|
|
|
225
214
|
let surfSel = NSSelectorFromString("framebufferSurface")
|
|
@@ -230,14 +219,11 @@ final class FrameCapture {
|
|
|
230
219
|
// don't spend cycles re-encoding the same pixels back-to-back from the
|
|
231
220
|
// frame-callback path. BUT: we must still re-emit at the idle floor
|
|
232
221
|
// (~5 fps) so that downstream consumers keep seeing a live stream —
|
|
233
|
-
// see the `
|
|
222
|
+
// see the `idleInterval` doc-comment for why that matters.
|
|
234
223
|
let key = ObjectIdentifier(desc)
|
|
235
224
|
let seed = IOSurfaceGetSeed(surface)
|
|
236
|
-
let nowMs = DispatchTime.now().uptimeNanoseconds / 1_000_000
|
|
237
|
-
let sinceLastMs = nowMs &- lastCaptureTimeMs
|
|
238
225
|
let seedChanged = lastSeeds[key] != seed
|
|
239
|
-
|
|
240
|
-
if frameCount > 0, !seedChanged, !idleRefreshDue { return }
|
|
226
|
+
if frameCount > 0, !seedChanged, !force { return }
|
|
241
227
|
lastSeeds[key] = seed
|
|
242
228
|
|
|
243
229
|
let w = IOSurfaceGetWidth(surface)
|
|
@@ -258,10 +244,11 @@ final class FrameCapture {
|
|
|
258
244
|
)
|
|
259
245
|
guard status == kCVReturnSuccess, let pb = pixelBuffer?.takeRetainedValue() else { return }
|
|
260
246
|
|
|
261
|
-
|
|
247
|
+
lastCaptureTime = .now
|
|
262
248
|
frameCount += 1
|
|
263
249
|
let timestamp = CMTime(value: CMTimeValue(frameCount), timescale: 60)
|
|
264
|
-
|
|
250
|
+
guard let copy = photocopier.copy(pb) else { return }
|
|
251
|
+
onFrame?(copy, timestamp)
|
|
265
252
|
}
|
|
266
253
|
|
|
267
254
|
func getScreenSize() -> (width: Int, height: Int)? {
|
|
@@ -308,3 +295,14 @@ final class FrameCapture {
|
|
|
308
295
|
})
|
|
309
296
|
}
|
|
310
297
|
}
|
|
298
|
+
|
|
299
|
+
@objc protocol FramebufferDescriptor {
|
|
300
|
+
@objc(registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:)
|
|
301
|
+
func registerScreenCallbacks(
|
|
302
|
+
uuid: UUID,
|
|
303
|
+
callbackQueue: DispatchQueue,
|
|
304
|
+
frameCallback: @convention(block) @escaping () -> Void,
|
|
305
|
+
surfacesChangedCallback: @convention(block) @escaping () -> Void,
|
|
306
|
+
propertiesChangedCallback: @convention(block) @escaping () -> Void
|
|
307
|
+
)
|
|
308
|
+
}
|
|
@@ -11,7 +11,10 @@ import VideoToolbox
|
|
|
11
11
|
/// The incoming buffer wraps SimulatorKit's live framebuffer IOSurface, which
|
|
12
12
|
/// SimulatorKit recycles in place — VT encodes asynchronously, so we deep-copy
|
|
13
13
|
/// into a private pooled buffer before submitting to avoid a torn frame race.
|
|
14
|
-
|
|
14
|
+
actor H264Encoder {
|
|
15
|
+
let queue = DispatchSerialQueue(label: "h264-encoder", qos: .userInteractive)
|
|
16
|
+
nonisolated var unownedExecutor: UnownedSerialExecutor { queue.asUnownedSerialExecutor() }
|
|
17
|
+
|
|
15
18
|
struct Encoded {
|
|
16
19
|
/// avcC parameter-set blob — emitted once on the first IDR per session.
|
|
17
20
|
let description: Data?
|
|
@@ -21,16 +24,11 @@ final class H264Encoder {
|
|
|
21
24
|
enum Kind { case keyframe, delta }
|
|
22
25
|
}
|
|
23
26
|
|
|
24
|
-
var onEncoded: ((Encoded) -> Void)?
|
|
25
|
-
|
|
26
|
-
private let lock = NSLock()
|
|
27
27
|
private var session: VTCompressionSession?
|
|
28
|
-
private var pool: CVPixelBufferPool?
|
|
29
28
|
private var width: Int32 = 0
|
|
30
29
|
private var height: Int32 = 0
|
|
31
30
|
private let fps: Int32
|
|
32
31
|
private var bitrate: Int
|
|
33
|
-
private let stateQueue = DispatchQueue(label: "H264Encoder.state")
|
|
34
32
|
private var emittedDescription = false
|
|
35
33
|
private var frameCount: Int64 = 0
|
|
36
34
|
|
|
@@ -44,8 +42,7 @@ final class H264Encoder {
|
|
|
44
42
|
}
|
|
45
43
|
|
|
46
44
|
/// Submit a frame. Returns immediately; `onEncoded` fires on VT's queue.
|
|
47
|
-
func encode(_ source: CVPixelBuffer, forceKeyframe: Bool = false
|
|
48
|
-
lock.lock()
|
|
45
|
+
func encode(_ source: CVPixelBuffer, forceKeyframe: Bool = false) async throws -> Encoded {
|
|
49
46
|
let w = Int32(CVPixelBufferGetWidth(source))
|
|
50
47
|
let h = Int32(CVPixelBufferGetHeight(source))
|
|
51
48
|
if session == nil || w != width || h != height {
|
|
@@ -53,10 +50,8 @@ final class H264Encoder {
|
|
|
53
50
|
height = h
|
|
54
51
|
rebuildSession()
|
|
55
52
|
}
|
|
56
|
-
guard let session
|
|
57
|
-
|
|
58
|
-
completion?()
|
|
59
|
-
return
|
|
53
|
+
guard let session else {
|
|
54
|
+
throw Errors.couldNotCreateSession
|
|
60
55
|
}
|
|
61
56
|
|
|
62
57
|
frameCount += 1
|
|
@@ -64,63 +59,39 @@ final class H264Encoder {
|
|
|
64
59
|
let frameProps: NSDictionary? = forceKeyframe
|
|
65
60
|
? [kVTEncodeFrameOptionKey_ForceKeyFrame: kCFBooleanTrue!] as NSDictionary
|
|
66
61
|
: nil
|
|
67
|
-
lock.unlock()
|
|
68
62
|
|
|
69
|
-
let
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
63
|
+
let buffer: CMSampleBuffer? = await withCheckedContinuation { continuation in
|
|
64
|
+
let status = VTCompressionSessionEncodeFrame(
|
|
65
|
+
session,
|
|
66
|
+
imageBuffer: source,
|
|
67
|
+
presentationTimeStamp: pts,
|
|
68
|
+
duration: .invalid,
|
|
69
|
+
frameProperties: frameProps,
|
|
70
|
+
infoFlagsOut: nil
|
|
71
|
+
) { @Sendable status, _, sampleBuffer in
|
|
72
|
+
guard status == noErr, let sb = sampleBuffer else {
|
|
73
|
+
continuation.resume(returning: nil)
|
|
74
|
+
return
|
|
75
|
+
}
|
|
76
|
+
continuation.resume(returning: sb)
|
|
77
|
+
}
|
|
78
|
+
if status != noErr {
|
|
79
|
+
continuation.resume(returning: nil)
|
|
80
|
+
}
|
|
83
81
|
}
|
|
82
|
+
guard let buffer else { throw Errors.encodingFailed }
|
|
83
|
+
return try extract(from: buffer)
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
func stop() {
|
|
87
|
-
lock.lock()
|
|
88
|
-
defer { lock.unlock() }
|
|
89
87
|
if let session {
|
|
90
88
|
VTCompressionSessionInvalidate(session)
|
|
91
89
|
self.session = nil
|
|
92
90
|
}
|
|
93
|
-
pool = nil
|
|
94
91
|
}
|
|
95
92
|
|
|
96
93
|
// MARK: - private
|
|
97
94
|
|
|
98
|
-
/// Deep-copy `source` (which wraps the recycled framebuffer IOSurface)
|
|
99
|
-
/// into a private pooled buffer that VT can hold past this call.
|
|
100
|
-
private func copyBuffer(_ source: CVPixelBuffer) -> CVPixelBuffer? {
|
|
101
|
-
guard let pool else { return nil }
|
|
102
|
-
var out: CVPixelBuffer?
|
|
103
|
-
guard CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &out) == kCVReturnSuccess,
|
|
104
|
-
let dst = out else { return nil }
|
|
105
|
-
|
|
106
|
-
CVPixelBufferLockBaseAddress(source, .readOnly)
|
|
107
|
-
CVPixelBufferLockBaseAddress(dst, [])
|
|
108
|
-
defer {
|
|
109
|
-
CVPixelBufferUnlockBaseAddress(dst, [])
|
|
110
|
-
CVPixelBufferUnlockBaseAddress(source, .readOnly)
|
|
111
|
-
}
|
|
112
|
-
guard let src = CVPixelBufferGetBaseAddress(source),
|
|
113
|
-
let dstAddr = CVPixelBufferGetBaseAddress(dst) else { return nil }
|
|
114
|
-
let srcStride = CVPixelBufferGetBytesPerRow(source)
|
|
115
|
-
let dstStride = CVPixelBufferGetBytesPerRow(dst)
|
|
116
|
-
let rows = CVPixelBufferGetHeight(source)
|
|
117
|
-
let copyBytes = min(srcStride, dstStride)
|
|
118
|
-
for row in 0..<rows {
|
|
119
|
-
memcpy(dstAddr + row * dstStride, src + row * srcStride, copyBytes)
|
|
120
|
-
}
|
|
121
|
-
return dst
|
|
122
|
-
}
|
|
123
|
-
|
|
124
95
|
private func rebuildSession() {
|
|
125
96
|
if let session {
|
|
126
97
|
VTCompressionSessionInvalidate(session)
|
|
@@ -173,43 +144,30 @@ final class H264Encoder {
|
|
|
173
144
|
}
|
|
174
145
|
VTCompressionSessionPrepareToEncodeFrames(sess)
|
|
175
146
|
session = sess
|
|
176
|
-
|
|
177
|
-
emittedDescription = false
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
// Pool feeding the deep-copy; BGRA matches the framebuffer surface.
|
|
181
|
-
let attrs: [String: Any] = [
|
|
182
|
-
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
|
183
|
-
kCVPixelBufferWidthKey as String: Int(width),
|
|
184
|
-
kCVPixelBufferHeightKey as String: Int(height),
|
|
185
|
-
kCVPixelBufferIOSurfacePropertiesKey as String: [:],
|
|
186
|
-
]
|
|
187
|
-
var newPool: CVPixelBufferPool?
|
|
188
|
-
CVPixelBufferPoolCreate(kCFAllocatorDefault, nil, attrs as CFDictionary, &newPool)
|
|
189
|
-
pool = newPool
|
|
147
|
+
emittedDescription = false
|
|
190
148
|
}
|
|
191
149
|
|
|
192
|
-
private func extract(from sample: CMSampleBuffer) -> Encoded
|
|
150
|
+
private func extract(from sample: CMSampleBuffer) throws -> Encoded {
|
|
193
151
|
let isKeyframe = !notSync(sample)
|
|
194
|
-
guard let dataBuf = CMSampleBufferGetDataBuffer(sample) else {
|
|
152
|
+
guard let dataBuf = CMSampleBufferGetDataBuffer(sample) else {
|
|
153
|
+
throw Errors.invalidSampleBuffer
|
|
154
|
+
}
|
|
195
155
|
|
|
196
156
|
var totalLength = 0
|
|
197
157
|
var dataPointer: UnsafeMutablePointer<Int8>?
|
|
198
158
|
guard CMBlockBufferGetDataPointer(
|
|
199
159
|
dataBuf, atOffset: 0, lengthAtOffsetOut: nil,
|
|
200
160
|
totalLengthOut: &totalLength, dataPointerOut: &dataPointer
|
|
201
|
-
) == noErr, let dataPointer else {
|
|
161
|
+
) == noErr, let dataPointer else {
|
|
162
|
+
throw Errors.invalidSampleBuffer
|
|
163
|
+
}
|
|
202
164
|
let avcc = Data(bytes: dataPointer, count: totalLength)
|
|
203
165
|
|
|
204
166
|
var description: Data?
|
|
205
167
|
if isKeyframe, let format = CMSampleBufferGetFormatDescription(sample) {
|
|
206
168
|
let nextDescription = avcCBlob(from: format)
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
emittedDescription = nextDescription != nil
|
|
210
|
-
return nextDescription != nil
|
|
211
|
-
}
|
|
212
|
-
if shouldEmit {
|
|
169
|
+
if !emittedDescription && nextDescription != nil {
|
|
170
|
+
emittedDescription = true
|
|
213
171
|
description = nextDescription
|
|
214
172
|
}
|
|
215
173
|
}
|
|
@@ -258,4 +216,10 @@ final class H264Encoder {
|
|
|
258
216
|
blob.append(contentsOf: pps)
|
|
259
217
|
return blob
|
|
260
218
|
}
|
|
219
|
+
|
|
220
|
+
enum Errors: Error {
|
|
221
|
+
case couldNotCreateSession
|
|
222
|
+
case encodingFailed
|
|
223
|
+
case invalidSampleBuffer
|
|
224
|
+
}
|
|
261
225
|
}
|