serve-sim 0.1.39 → 0.1.40-beta.73.1

package/Sources/SimCameraHelper/build.sh

@@ -20,6 +20,7 @@ xcrun --sdk macosx clang \
     -framework CoreImage \
     -framework CoreText \
     -framework ImageIO \
+    -framework IOSurface \
     -framework Accelerate \
     -O2 \
     -o "$BIN" \

package/Sources/SimCameraHelper/main.m

@@ -29,6 +29,7 @@
 #import <CoreImage/CoreImage.h>
 #import <Accelerate/Accelerate.h>
 #import <ImageIO/ImageIO.h>
+#import <IOSurface/IOSurface.h>
 
 #include <fcntl.h>
 #include <signal.h>
@@ -44,7 +45,9 @@
 #pragma mark - Globals (shm + writer)
 
 static SimCamShmHeader *gHeader = NULL;
-static uint8_t *gPixels = NULL;
+static SimCamSurfaceTable *gSurfaceTable = NULL;
+static IOSurfaceRef gSurfaces[SIMCAM_SURFACE_RING];
+static uint32_t gWriteIndex = 0;            // last ring slot rendered into
 static uint32_t gWidth = SIMCAM_DEFAULT_WIDTH;
 static uint32_t gHeight = SIMCAM_DEFAULT_HEIGHT;
 static const char *gShmName = NULL;
@@ -65,15 +68,48 @@ static uint8_t ParseMirrorCode(NSString *mode);
 static NSString *MirrorName(uint8_t code);
 
 // Publish a fully-prepared BGRA frame (gWidth x gHeight, packed at gWidth*4
-// bytes per row) to the shm region. Writers MUST go through this so seq/ts
-// stay coherent for the dylib's tear-detection check.
+// bytes per row) into the next free ring surface. Writers MUST go through this
+// so latestIndex/frameSeq stay coherent for the dylib's tear-detection check.
 static void PublishFrame(const uint8_t *bgra) {
-    if (!gHeader || !gPixels || !bgra) return;
-    memcpy(gPixels, bgra, (size_t)gWidth * gHeight * 4);
+    if (!gHeader || !gSurfaceTable || !bgra) return;
+    uint32_t count = gSurfaceTable->surfaceCount;
+    if (count == 0) return;
+
+    // Render into a surface the reader isn't holding and isn't the one it last
+    // published, so an in-flight frame is never overwritten mid-read.
+    uint32_t latest = gSurfaceTable->latestIndex;
+    uint32_t idx = gWriteIndex;
+    BOOL found = NO;
+    for (uint32_t tries = 0; tries < count; tries++) {
+        idx = (idx + 1) % count;
+        if (idx == latest) continue;
+        if (!IOSurfaceIsInUse(gSurfaces[idx])) {
+            found = YES;
+            break;
+        }
+    }
+    if (!found) return;
+    gWriteIndex = idx;
+
+    IOSurfaceRef surface = gSurfaces[idx];
+    IOSurfaceLock(surface, 0, NULL);
+    uint8_t *dst = (uint8_t *)IOSurfaceGetBaseAddress(surface);
+    size_t dstStride = IOSurfaceGetBytesPerRow(surface);
+    size_t srcStride = (size_t)gWidth * 4;
+    if (dstStride == srcStride) {
+        memcpy(dst, bgra, srcStride * gHeight);
+    } else {
+        for (uint32_t y = 0; y < gHeight; y++) {
+            memcpy(dst + (size_t)y * dstStride, bgra + (size_t)y * srcStride, srcStride);
+        }
+    }
+    IOSurfaceUnlock(surface, 0, NULL);
+
+    gSurfaceTable->latestIndex = idx;
     gHeader->timestampNs = MachAbsToNs(mach_absolute_time());
     atomic_thread_fence(memory_order_release);
     uint64_t next = atomic_fetch_add(&gFrameSeq, 1) + 1;
-    gHeader->frameSeq = next;
+    atomic_store_explicit(&gHeader->frameSeq, next, memory_order_release);
 }
 
 #pragma mark - Source pipeline (start / stop / switch)
@@ -754,7 +790,41 @@ static void ListDevices(void) {
     }
 }
 
-static int OpenShm(const char *name, size_t size) {
+// Allocate the IOSurface ring and record their global IDs in the table.
+static BOOL CreateSurfaces(void) {
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+    NSDictionary *props = @{
+        (id)kIOSurfaceWidth: @(gWidth),
+        (id)kIOSurfaceHeight: @(gHeight),
+        (id)kIOSurfaceBytesPerElement: @4,
+        (id)kIOSurfacePixelFormat: @((uint32_t)kCVPixelFormatType_32BGRA),
+        // Global so the simulator process can resolve the surface by ID.
+        (id)kIOSurfaceIsGlobal: @YES,
+    };
+#pragma clang diagnostic pop
+    for (uint32_t i = 0; i < SIMCAM_SURFACE_RING; i++) {
+        IOSurfaceRef s = IOSurfaceCreate((__bridge CFDictionaryRef)props);
+        if (!s) {
+            fprintf(stderr, "[serve-sim-camera] IOSurfaceCreate failed at %u\n", i);
+            return NO;
+        }
+        gSurfaces[i] = s;
+        gSurfaceTable->ids[i] = IOSurfaceGetID(s);
+    }
+    gSurfaceTable->surfaceCount = SIMCAM_SURFACE_RING;
+    gSurfaceTable->latestIndex = 0;
+    return YES;
+}
+
+static void ReleaseSurfaces(void) {
+    for (uint32_t i = 0; i < SIMCAM_SURFACE_RING; i++) {
+        if (gSurfaces[i]) { CFRelease(gSurfaces[i]); gSurfaces[i] = NULL; }
+    }
+}
+
+static int OpenShm(const char *name) {
+    size_t size = (size_t)SimCamControlSize();
     shm_unlink(name);
     int fd = shm_open(name, O_CREAT | O_RDWR, 0644);
     if (fd < 0) { perror("shm_open"); return -1; }
@@ -762,14 +832,15 @@ static int OpenShm(const char *name, size_t size) {
     void *map = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
     if (map == MAP_FAILED) { perror("mmap"); close(fd); return -1; }
     gHeader = (SimCamShmHeader *)map;
-    gPixels = (uint8_t *)map + sizeof(SimCamShmHeader);
-    memset(gHeader, 0, sizeof(*gHeader));
+    gSurfaceTable = (SimCamSurfaceTable *)((uint8_t *)map + sizeof(SimCamShmHeader));
+    memset(map, 0, size);
+    if (!CreateSurfaces()) { close(fd); return -1; }
     gHeader->magic = SIMCAM_SHM_MAGIC;
-    gHeader->version = 1;
+    gHeader->version = 2;
     gHeader->width = gWidth;
     gHeader->height = gHeight;
     gHeader->pixelFormat = SIMCAM_PIXEL_BGRA;
-    gHeader->bytesPerRow = gWidth * 4;
+    gHeader->bytesPerRow = (uint32_t)IOSurfaceGetBytesPerRow(gSurfaces[0]);
     gHeader->pixelByteSize = (uint64_t)gWidth * gHeight * 4;
     gHeader->mirrorMode = SIMCAM_MIRROR_UNSET; // dylib falls back to env
     return fd;
@@ -824,10 +895,9 @@ int main(int argc, const char *argv[]) {
             // (no-op marker; --device implies webcam below if user intended it)
         }
 
-        size_t shmSize = (size_t)SimCamShmSizeFor(gWidth, gHeight);
-        if (OpenShm(gShmName, shmSize) < 0) return 1;
-        fprintf(stderr, "[serve-sim-camera] shm \"%s\" %zu bytes (%ux%u BGRA)\n",
-                gShmName, shmSize, gWidth, gHeight);
+        if (OpenShm(gShmName) < 0) return 1;
+        fprintf(stderr, "[serve-sim-camera] shm \"%s\" + %u IOSurfaces (%ux%u BGRA)\n",
+                gShmName, SIMCAM_SURFACE_RING, gWidth, gHeight);
 
         gSourceQueue = dispatch_queue_create("simcam.helper.source", DISPATCH_QUEUE_SERIAL);
 
@@ -863,6 +933,8 @@ int main(int argc, const char *argv[]) {
         if (gControlListenFd >= 0) { close(gControlListenFd); if (socketPath) unlink(socketPath); }
         StopPlaceholderSource();
         StopWebcamSource();
+        StopVideoSource();
+        ReleaseSurfaces();
         if (gShmName) shm_unlink(gShmName);
         fprintf(stderr, "[serve-sim-camera] stopped\n");
         return 0;

package/Sources/SimCameraInjector/SimCamFrameSource.m

@@ -6,6 +6,7 @@
 #import <CoreImage/CoreImage.h>
 #import <CoreMedia/CoreMedia.h>
 #import <CoreVideo/CoreVideo.h>
+#import <IOSurface/IOSurfaceRef.h>
 #import <UIKit/UIKit.h>
 #import <QuartzCore/QuartzCore.h>
 #import <objc/runtime.h>
@@ -26,8 +27,8 @@ static size_t kFrameHeight = 720;
 static const double kFrameRate = 30.0;
 
 static SimCamShmHeader *gShmHeader = NULL;
-static const uint8_t *gShmPixels = NULL;
-static size_t gShmTotalSize = 0;
+static SimCamSurfaceTable *gSurfaceTable = NULL;
+static IOSurfaceRef gSurfaces[SIMCAM_SURFACE_RING];  // resolved from global IDs
 static uint64_t gLastSeenSeq = 0;
 
 #pragma mark - Last-frame cache
@@ -236,38 +237,37 @@ static CGImageRef SimCamAcquireCachedCGImage(void) CF_RETURNS_RETAINED {
     });
 }
 
-- (CVPixelBufferRef)newPixelBufferFromShm CF_RETURNS_RETAINED {
-    return [self newPixelBufferFromShmForceFresh:NO];
+- (CVPixelBufferRef)newPixelBufferFromSurface CF_RETURNS_RETAINED {
+    return [self newPixelBufferFromSurfaceForceFresh:NO];
 }
 
-- (CVPixelBufferRef)newPixelBufferFromShmForceFresh:(BOOL)force CF_RETURNS_RETAINED {
-    if (!gShmHeader || !gShmPixels) return NULL;
+// Wrap the latest shared IOSurface as a CVPixelBuffer — zero copy. Holding the
+// pixel buffer keeps the surface in use, so the host writer renders into a
+// different ring slot until we release it.
+- (CVPixelBufferRef)newPixelBufferFromSurfaceForceFresh:(BOOL)force CF_RETURNS_RETAINED {
+    if (!gShmHeader || !gSurfaceTable) return NULL;
     if (gShmHeader->magic != SIMCAM_SHM_MAGIC) return NULL;
-    uint64_t seqA = gShmHeader->frameSeq;
+    uint64_t seqA = atomic_load_explicit(&gShmHeader->frameSeq, memory_order_acquire);
     if (seqA == 0) return NULL;
     if (!force && seqA == gLastSeenSeq) return NULL;
-    uint32_t w = gShmHeader->width;
-    uint32_t h = gShmHeader->height;
-    uint32_t bpr = gShmHeader->bytesPerRow;
-    if (!w || !h || bpr < w * 4) return NULL;
-    if (sizeof(SimCamShmHeader) + (size_t)bpr * h > gShmTotalSize) return NULL;
+
+    uint32_t count = gSurfaceTable->surfaceCount;
+    if (count == 0 || count > SIMCAM_SURFACE_RING) return NULL;
+    uint32_t idx = gSurfaceTable->latestIndex;
+    if (idx >= count) return NULL;
+    IOSurfaceRef surface = gSurfaces[idx];
+    if (!surface) {
+        simcam_log(@"missing IOSurface at latest index %u/%u", idx, count);
+        return NULL;
+    }
 
     CVPixelBufferRef pb = NULL;
     NSDictionary *attrs = @{ (id)kCVPixelBufferIOSurfacePropertiesKey: @{} };
-    CVReturn r = CVPixelBufferCreate(kCFAllocatorDefault, w, h,
-        kCVPixelFormatType_32BGRA, (__bridge CFDictionaryRef)attrs, &pb);
+    CVReturn r = CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, surface,
+        (__bridge CFDictionaryRef)attrs, &pb);
     if (r != kCVReturnSuccess || !pb) return NULL;
-    CVPixelBufferLockBaseAddress(pb, 0);
-    uint8_t *dst = (uint8_t *)CVPixelBufferGetBaseAddress(pb);
-    size_t dstBpr = CVPixelBufferGetBytesPerRow(pb);
-    size_t copyBpr = MIN((size_t)bpr, dstBpr);
-    for (uint32_t y = 0; y < h; y++) {
-        memcpy(dst + y * dstBpr, gShmPixels + y * bpr, copyBpr);
-    }
-    CVPixelBufferUnlockBaseAddress(pb, 0);
 
-    atomic_thread_fence(memory_order_acquire);
-    uint64_t seqB = gShmHeader->frameSeq;
+    uint64_t seqB = atomic_load_explicit(&gShmHeader->frameSeq, memory_order_acquire);
     if (!force && seqA != seqB) {
         CVPixelBufferRelease(pb);
         return NULL;
@@ -277,7 +277,7 @@ static CGImageRef SimCamAcquireCachedCGImage(void) CF_RETURNS_RETAINED {
 }
 
 - (CVPixelBufferRef)currentPixelBuffer CF_RETURNS_RETAINED {
-    CVPixelBufferRef pb = [self newPixelBufferFromShmForceFresh:YES];
+    CVPixelBufferRef pb = [self newPixelBufferFromSurfaceForceFresh:YES];
     if (!pb) pb = [self newPixelBufferFromImage];
     return pb;
 }
@@ -350,7 +350,7 @@ static CGImageRef SimCamAcquireCachedCGImage(void) CF_RETURNS_RETAINED {
 }
 
 - (CMSampleBufferRef)newSampleBufferAtTime:(CMTime)pts CF_RETURNS_RETAINED {
-    CVPixelBufferRef pb = [self newPixelBufferFromShm];
+    CVPixelBufferRef pb = [self newPixelBufferFromSurface];
     if (!pb) pb = [self newPixelBufferFromImage];
     if (pb) {
         SimCamCacheFrame(pb);
@@ -362,7 +362,7 @@ static CGImageRef SimCamAcquireCachedCGImage(void) CF_RETURNS_RETAINED {
         dispatch_once(&logOnce, ^{
             simcam_log(@"no-signal fallback: shm=%@ frameSeq=%llu cache=empty",
                 gShmHeader ? @"attached" : @"unattached",
-                (unsigned long long)(gShmHeader ? gShmHeader->frameSeq : 0));
+                (unsigned long long)(gShmHeader ? atomic_load_explicit(&gShmHeader->frameSeq, memory_order_acquire) : 0));
         });
         pb = [self newPixelBufferNoSignal];
     }
@@ -518,13 +518,14 @@ void SimCamFrameSourceOpenShmIfRequested(void) {
         simcam_log(@"shm_open(%s) failed: %s", shmName, strerror(errno));
         return;
     }
+    size_t size = (size_t)SimCamControlSize();
     struct stat st;
-    if (fstat(fd, &st) < 0 || st.st_size < (off_t)sizeof(SimCamShmHeader)) {
+    if (fstat(fd, &st) < 0 || (size_t)st.st_size < size) {
         simcam_log(@"shm fstat failed or too small");
         close(fd);
         return;
     }
-    void *map = mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_SHARED, fd, 0);
+    void *map = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
     close(fd);
     if (map == MAP_FAILED) {
         simcam_log(@"shm mmap failed: %s", strerror(errno));
@@ -533,14 +534,44 @@ void SimCamFrameSourceOpenShmIfRequested(void) {
     SimCamShmHeader *hdr = (SimCamShmHeader *)map;
     if (hdr->magic != SIMCAM_SHM_MAGIC) {
         simcam_log(@"shm magic mismatch: 0x%x", hdr->magic);
-        munmap(map, (size_t)st.st_size);
+        munmap(map, size);
+        return;
+    }
+    SimCamSurfaceTable *table =
+        (SimCamSurfaceTable *)((uint8_t *)map + sizeof(SimCamShmHeader));
+    uint32_t count = table->surfaceCount;
+    if (count == 0 || count > SIMCAM_SURFACE_RING) {
+        simcam_log(@"shm surface table invalid (count=%u)", count);
+        munmap(map, size);
+        return;
+    }
+
+    // Resolve each global IOSurface ID to a local reference.
+    uint32_t resolved = 0;
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+    for (uint32_t i = 0; i < count; i++) {
+        IOSurfaceRef s = IOSurfaceLookup(table->ids[i]);
+        gSurfaces[i] = s;
+        if (s) resolved++;
+    }
+#pragma clang diagnostic pop
+    if (resolved != count) {
+        for (uint32_t i = 0; i < count; i++) {
+            if (gSurfaces[i]) {
+                CFRelease(gSurfaces[i]);
+                gSurfaces[i] = NULL;
+            }
+        }
+        simcam_log(@"only %u/%u IOSurfaces resolved from shm \"%s\"", resolved, count, shmName);
+        munmap(map, size);
         return;
     }
+
     gShmHeader = hdr;
-    gShmPixels = (const uint8_t *)map + sizeof(SimCamShmHeader);
-    gShmTotalSize = (size_t)st.st_size;
+    gSurfaceTable = table;
     kFrameWidth = hdr->width;
     kFrameHeight = hdr->height;
-    simcam_log(@"shm \"%s\" attached (%ux%u, %llu bytes)",
-               shmName, hdr->width, hdr->height, (unsigned long long)st.st_size);
+    simcam_log(@"shm \"%s\" attached (%ux%u, %u/%u IOSurfaces resolved)",
+               shmName, hdr->width, hdr->height, resolved, count);
 }

package/Sources/SimCameraInjector/build.sh

@@ -25,6 +25,7 @@ xcrun --sdk iphonesimulator clang \
     -framework CoreMotion \
     -framework CoreVideo \
     -framework CoreGraphics \
+    -framework IOSurface \
     -framework QuartzCore \
     -install_name "@rpath/libSimCameraInjector.dylib" \
     -o "$DYLIB" \

package/Sources/SimCameraInjector/include/SimCamShared.h

@@ -1,27 +1,41 @@
-// Shared memory wire format for serve-sim's simulator camera feed.
+// Wire format for serve-sim's simulator camera feed.
 //
-// One process (the host helper, running on macOS) captures webcam frames and
-// writes them into a POSIX shared-memory region. The injected dylib inside
-// the simulator app maps the same region and dispatches the latest frame to
-// AVFoundation delegates / preview layers.
+// Frames travel over a small ring of IOSurfaces shared by global surface ID.
+// The host helper (running on macOS) renders BGRA frames straight into the
+// surfaces; the injected dylib inside the simulator app looks the surfaces up
+// by ID and wraps the latest one as a CVPixelBuffer — no per-frame pixel copy.
 //
-// Format: BGRA, top-down, no padding except whatever bytesPerRow says.
+// A tiny POSIX shared-memory region acts as the control channel: it carries
+// the surface IDs, which surface holds the newest frame, the frame sequence
+// number, and the mirror mode. The pixels themselves live in the IOSurfaces,
+// not in this region.
 //
-// Synchronization is intentionally lock-free and lossy: the writer bumps
-// `frameSeq` last after writing pixels; the reader samples `frameSeq` before
-// and after copying and discards the read if they disagree. A single dropped
-// frame is fine for a 30 fps camera.
+// Surfaces are BGRA. Row stride comes from IOSurfaceGetBytesPerRow (it may be
+// padded beyond width*4); `bytesPerRow` below mirrors that actual stride.
+//
+// Synchronization is lock-free and lossy. The writer renders into a surface
+// the reader is not holding, points `latestIndex` at it, then bumps `frameSeq`
+// last. The reader samples `frameSeq` before and after wrapping the surface and
+// retries on the next tick if they disagree. A single dropped frame is fine for
+// a 30 fps camera.
 
 #ifndef SIM_CAM_SHARED_H
 #define SIM_CAM_SHARED_H
 
+#include <stddef.h>
 #include <stdint.h>
+#include <stdatomic.h>
 
 #define SIMCAM_SHM_MAGIC      0x53434D31u  // 'SCM1'
 #define SIMCAM_PIXEL_BGRA     0u
 #define SIMCAM_DEFAULT_WIDTH  1280u
 #define SIMCAM_DEFAULT_HEIGHT 720u
 
+// Number of IOSurfaces in the ring. The writer keeps off whichever surface the
+// reader most recently published, so a few buffers absorb a reader that holds
+// a frame for a tick or two without tearing.
+#define SIMCAM_SURFACE_RING   4u
+
 // Mirror mode codes for SimCamShmHeader.mirrorMode.
 // "Unset" lets the dylib fall back to its env-var configuration (back-compat
 // with hosts that don't write the byte).
@@ -30,26 +44,36 @@
 #define SIMCAM_MIRROR_ON      1
 #define SIMCAM_MIRROR_OFF     2
 
-// Header is 64 bytes — keeps pixel data 16-byte aligned.
-typedef struct __attribute__((packed)) {
+// Control header is 64 bytes. The surface-ID table follows immediately after.
+typedef struct {
     uint32_t magic;        // SIMCAM_SHM_MAGIC
     uint32_t version;      // bumps on layout change
     uint32_t width;
     uint32_t height;
     uint32_t pixelFormat;  // SIMCAM_PIXEL_BGRA
-    uint32_t bytesPerRow;
-    uint64_t pixelByteSize;
-    uint64_t frameSeq;     // written LAST; readers check tearing via re-read
+    uint32_t bytesPerRow;  // actual IOSurface row stride (may exceed width*4)
+    uint64_t pixelByteSize;// logical frame size: width*height*4
+    _Atomic uint64_t frameSeq; // written LAST with release; readers acquire-load
     uint64_t timestampNs;  // mach_absolute_time-based, host monotonic
     uint8_t  mirrorMode;   // SIMCAM_MIRROR_*; UNSET = ignore (use env)
     uint8_t  reserved[15];
 } SimCamShmHeader;
 
 _Static_assert(sizeof(SimCamShmHeader) == 64, "SimCamShmHeader must be 64 bytes");
+_Static_assert(offsetof(SimCamShmHeader, frameSeq) == 32, "frameSeq offset must stay stable");
+_Static_assert(offsetof(SimCamShmHeader, mirrorMode) == 48, "mirrorMode offset must stay stable");
+
+// Ring of global IOSurface IDs, written once at startup. `latestIndex` is
+// updated each frame (before frameSeq) to point at the freshest surface.
+typedef struct __attribute__((packed)) {
+    uint32_t surfaceCount;                 // valid entries in ids[]
+    uint32_t latestIndex;                  // ids[] slot holding the newest frame
+    uint32_t ids[SIMCAM_SURFACE_RING];     // global IOSurface IDs
+} SimCamSurfaceTable;
 
-// Total shm size for given dimensions: header + width*height*4
-static inline uint64_t SimCamShmSizeFor(uint32_t w, uint32_t h) {
-    return (uint64_t)sizeof(SimCamShmHeader) + (uint64_t)w * (uint64_t)h * 4ull;
+// Total control-region size: header + surface table.
+static inline uint64_t SimCamControlSize(void) {
+    return (uint64_t)sizeof(SimCamShmHeader) + (uint64_t)sizeof(SimCamSurfaceTable);
 }
 
 #endif