replicas-computer 0.3.7 → 0.3.8

package/dist/stream-viewer.js

@@ -736,7 +736,7 @@ var updateDiagnostics = (stats) => {
     css: `${Math.round(bounds.width)}\xD7${Math.round(bounds.height)}`,
     physical: `${Math.round(bounds.width * devicePixelRatio)}\xD7${Math.round(bounds.height * devicePixelRatio)} @${devicePixelRatio}x`
   };
-  Object.assign(state, { transport, transportReason, socketRttMs: socketRtt, view, route, dims, ...stats });
+  Object.assign(state, { transport, transportReason, socketRttMs: socketRtt, view, route, dims, decodeMs: stats.decodeMs, receiverProcessingMs: stats.receiverProcessingMs, ...stats });
   if (!showDiagnostics) return;
   const network = stats.lossPercent === void 0 ? `${stats.decodeQueue} queued` : `${stats.lossPercent.toFixed(1)}% loss \xB7 ${(stats.playbackDroppedPercent ?? 0).toFixed(1)}% playback drops \xB7 ${stats.nacks ?? 0} nack \xB7 ${stats.freezes ?? 0} freezes \xB7 ${(stats.jitterBufferMs ?? 0).toFixed(0)} ms buffer`;
   const encoder = telemetry?.encoder;
@@ -745,6 +745,7 @@ var updateDiagnostics = (stats) => {
     `${transport ?? "connecting"}${transportReason ? ` (${transportReason})` : ""} \xB7 ${route.path || "\u2014"} \xB7 ${route.codec || config?.videoCodec || ""} \xB7 ${view} view`,
     `source ${dims.source} \xB7 decoded ${dims.decoded || "\u2014"} \xB7 css ${dims.css} \xB7 device ${dims.physical}`,
     `${route.displayedFps.toFixed(1)} displayed/s \xB7 ${stats.videoFps.toFixed(1)} decoded/s \xB7 ${route.kbps.toFixed(0)} kbps \xB7 ${stats.rttMs.toFixed(0)} ms rtt \xB7 ${socketRtt.toFixed(0)} ms via proxy \xB7 ${network}`,
+    transport === "webrtc" ? `receiver: ${stats.decodeMs?.toFixed(2) ?? "\u2014"} ms decode \xB7 ${stats.receiverProcessingMs?.toFixed(2) ?? "\u2014"} ms receive \u2192 decoded (includes buffer + decode; excludes display)` : "",
     encoder ? `encoder ${encoder.profile} crf ${encoder.crf} \xB7 ${encoder.codecProfile} ${encoder.pixelFormat} ${encoder.color} \xB7 rung ${encoder.rung}/${encoder.rungs - 1} cap ${encoder.kbpsCap} kbps${encoder.adaptive ? "" : " pinned"} \xB7 keyframe every ${encoder.keyframeMs} ms \xB7 ${encoder.fps} fps ${encoder.kbps} kbps \xB7 ${encoder.preset}` : "",
     telemetry && server ? `server cpu: encoder ${telemetry.cpu.encoderPercent}% \xB7 stream ${telemetry.cpu.serverPercent}% \xB7 load ${telemetry.cpu.load} on ${telemetry.cpu.cores} cores \xB7 budget ${server.budgetKbps} kbps sent ${server.sentKbps} \xB7 queue ${server.queueMs} ms ${server.queuedPackets} pkt \xB7 stale ${server.staleEvents} dropped ${server.droppedFrames} \xB7 repairs ${server.repairsSent}/${server.repairsSkipped} skipped \xB7 copies ${server.duplicatesSent}${server.protecting ? " on" : ""} \xB7 remb ${server.rembKbps ?? "\u2014"}` : "",
     transport === "websocket" && server ? `socket: ${Math.round(server.bufferedBytes / 1024)} KiB unsent \xB7 ${server.withheldFrames} deltas withheld${server.waitingForKeyframe ? " \xB7 awaiting keyframe" : ""} \xB7 ${decoderResets} decoder resets` : "",
@@ -773,6 +774,9 @@ var reportWebRtcStats = async (pc, stats) => {
   const now = performance.now();
   const playback = screen.getVideoPlaybackQuality?.();
   const counters = {
+    id: video?.id,
+    totalDecodeTime: video?.totalDecodeTime,
+    totalProcessingDelay: video?.totalProcessingDelay,
     framesDecoded: video?.framesDecoded ?? 0,
     totalVideoFrames: playback?.totalVideoFrames ?? 0,
     droppedVideoFrames: playback?.droppedVideoFrames ?? 0,
@@ -790,6 +794,7 @@ var reportWebRtcStats = async (pc, stats) => {
     clearTimeout(webrtcDeadline);
     webrtcDeadline = void 0;
   }
+  if (inbound && (inbound.id !== counters.id || counters.framesDecoded < inbound.framesDecoded)) inbound = void 0;
   if (!inbound) inboundHistory = [];
   while (inboundHistory.length > 1 && inboundHistory[1].at <= now - 1e3) inboundHistory.shift();
   const oldest = inboundHistory[0];
@@ -803,7 +808,17 @@ var reportWebRtcStats = async (pc, stats) => {
     const received = counters.packetsReceived - inbound.packetsReceived;
     stats.videoPacketsPerSecond = Math.max(0, received) / seconds;
     const emitted = counters.jitterBufferEmittedCount - inbound.jitterBufferEmittedCount;
-    stats.videoFps = (counters.framesDecoded - inbound.framesDecoded) / seconds;
+    const decoded = counters.framesDecoded - inbound.framesDecoded;
+    stats.videoFps = decoded / seconds;
+    if (decoded > 0) {
+      for (const [counter, metric] of [["totalDecodeTime", "decodeMs"], ["totalProcessingDelay", "receiverProcessingMs"]]) {
+        const current = counters[counter];
+        const previous = inbound[counter];
+        if (current !== void 0 && previous !== void 0 && Number.isFinite(current) && Number.isFinite(previous) && previous >= 0 && current >= previous) {
+          stats[metric] = (current - previous) * 1e3 / decoded;
+        }
+      }
+    }
     const total = counters.totalVideoFrames - inbound.totalVideoFrames;
     const dropped = Math.max(0, counters.droppedVideoFrames - inbound.droppedVideoFrames);
     if (playback) stats.playbackDroppedPercent = total > 0 ? dropped * 100 / total : 0;

package/native/go.mod

@@ -0,0 +1,3 @@
+module github.com/replicas-group/replicas/computer-use/native
+
+go 1.27.1

package/native/internal/capture/fixture.go

@@ -0,0 +1,130 @@
+//go:build libyuv && capturetest
+
+package capture
+
+/*
+#define XUTIL_DEFINE_FUNCTIONS
+#include <X11/Xlib.h>
+#include <X11/Xutil.h>
+#include <libavutil/frame.h>
+#include <libyuv/convert.h>
+#include <libyuv/convert_from_argb.h>
+#include <libyuv/row.h>
+*/
+import "C"
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"runtime"
+	"unsafe"
+)
+
+// cgo cannot be imported directly from a _test.go file.
+func checkPartialCapture(width, height int, i420, bt709 bool) error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	d, err := newDesktop(Options{Width: width, Height: height}, true)
+	if err != nil {
+		return err
+	}
+	defer d.close()
+	if d.attributes.width != 128 || d.attributes.height != 96 {
+		return errors.New("capture tests require the isolated 128x96 display from test:capture")
+	}
+	e := &encoder{}
+	defer e.close()
+	e.matrix, err = newColorMatrix(bt709)
+	if err != nil {
+		return err
+	}
+	e.frame = C.av_frame_alloc()
+	if e.frame == nil {
+		return errors.New("cannot allocate test frame")
+	}
+	e.frame.width, e.frame.height = C.int(width), C.int(height)
+	e.frame.format = C.AV_PIX_FMT_YUV444P
+	chroma := 0
+	if i420 {
+		e.frame.format = C.AV_PIX_FMT_YUV420P
+		chroma = 1
+	}
+	if err = avCheck(C.av_frame_get_buffer(e.frame, 32)); err != nil {
+		return err
+	}
+	s := Stream{desktop: d, encoder: e}
+	gc := C.XCreateGC(d.display, d.root, 0, nil)
+	defer C.XFreeGC(d.display, gc)
+	rectangles := [][4]int{
+		{0, 0, width, height},
+		{0, 0, 1, 1},
+		{width - 1, height - 1, 1, 1},
+		{width - 3, 1, 9, 3},
+		{1, height - 3, 3, 9},
+		{3, 5, 7, 11},
+		{width/2 + 1, height/2 + 1, 1, 1},
+		{0, 0, width, height},
+	}
+	size := width * height
+	reference := make([]byte, size*3)
+	strides := [3]C.int{C.int(width), C.int(width >> chroma), C.int(width >> chroma)}
+	partial, resized := 0, 0
+	previousWidth, previousHeight := width, height
+	for step := 0; step < 128; step++ {
+		r := rectangles[step%len(rectangles)]
+		C.XSetForeground(d.display, gc, C.ulong((step+1)*0x915b37&0xffffff))
+		C.XFillRectangle(d.display, d.root, gc, C.int(r[0]), C.int(r[1]), C.uint(r[2]), C.uint(r[3]))
+		C.XSync(d.display, 0)
+		changed, err := s.Changed()
+		if err != nil || !changed {
+			return fmt.Errorf("step %d: missing damage or error: %v", step, err)
+		}
+		if _, _, err = s.Capture(step == 0); err != nil {
+			return err
+		}
+		if int(d.image.width) < width || int(d.image.height) < height {
+			partial++
+		}
+		if int(d.image.width) != previousWidth || int(d.image.height) != previousHeight {
+			resized++
+		}
+		previousWidth, previousHeight = int(d.image.width), int(d.image.height)
+		full := C.XGetImage(d.display, d.root, 0, 0, C.uint(width), C.uint(height), ^C.ulong(0), C.ZPixmap)
+		if full == nil {
+			return errors.New("cannot capture reference image")
+		}
+		planes := [3]*C.uint8_t{
+			(*C.uint8_t)(unsafe.Pointer(&reference[0])),
+			(*C.uint8_t)(unsafe.Pointer(&reference[size])),
+			(*C.uint8_t)(unsafe.Pointer(&reference[size*2])),
+		}
+		var code C.int
+		if i420 {
+			code = C.ARGBToI420Matrix((*C.uint8_t)(unsafe.Pointer(full.data)), full.bytes_per_line, planes[0], strides[0], planes[1], strides[1], planes[2], strides[2], e.matrix.pointer, C.int(width), C.int(height))
+		} else {
+			code = C.ARGBToI444Matrix((*C.uint8_t)(unsafe.Pointer(full.data)), full.bytes_per_line, planes[0], strides[0], planes[1], strides[1], planes[2], strides[2], e.matrix.pointer, C.int(width), C.int(height))
+		}
+		C.XDestroyImage(full)
+		if err = avCheck(code); err != nil {
+			return err
+		}
+		for plane := 0; plane < 3; plane++ {
+			rows := height
+			if plane != 0 {
+				rows >>= chroma
+			}
+			for row := 0; row < rows; row++ {
+				expected := reference[size*plane+row*int(strides[plane]) : size*plane+(row+1)*int(strides[plane])]
+				actual := unsafe.Slice((*byte)(unsafe.Add(unsafe.Pointer(e.frame.data[plane]), row*int(e.frame.linesize[plane]))), int(strides[plane]))
+				if !bytes.Equal(expected, actual) {
+					return fmt.Errorf("step %d rect %v: plane %d row %d differs from full conversion", step, r, plane, row)
+				}
+			}
+		}
+	}
+	if partial < 64 || resized < 64 {
+		return fmt.Errorf("partial path not exercised: partial=%d resized=%d", partial, resized)
+	}
+	return nil
+}

package/native/internal/capture/native.go

@@ -0,0 +1,462 @@
+package capture
+
+/*
+#cgo pkg-config: x11 xext xdamage xfixes libavcodec libavformat libavutil libswscale
+#include "native.h"
+*/
+import "C"
+import (
+	"errors"
+	"fmt"
+	"runtime"
+	"strconv"
+	"syscall"
+	"unsafe"
+)
+
+func avCheck(code C.int) error {
+	if code >= 0 {
+		return nil
+	}
+	var message [128]C.char
+	C.av_strerror(code, &message[0], C.size_t(len(message)))
+	return errors.New(C.GoString(&message[0]))
+}
+
+type desktop struct {
+	display       *C.Display
+	root          C.Window
+	attributes    C.XWindowAttributes
+	damage        C.Damage
+	damageEvent   C.int
+	region        C.XserverRegion
+	image         *C.XImage
+	shm           *C.XShmSegmentInfo
+	attached      bool
+	width, height int
+}
+
+func newDesktop(o Options, partial bool) (d *desktop, err error) {
+	d = &desktop{width: o.Width, height: o.Height}
+	defer func() {
+		if err != nil {
+			d.close()
+		}
+	}()
+	C.capture_install_handler()
+	d.display = C.XOpenDisplay(nil)
+	if d.display == nil {
+		return d, errors.New("cannot open display")
+	}
+	d.root = C.XDefaultRootWindow(d.display)
+	if C.XGetWindowAttributes(d.display, d.root, &d.attributes) == 0 {
+		return d, errors.New("cannot read display dimensions")
+	}
+	if o.Width > int(d.attributes.width) || o.Height > int(d.attributes.height) {
+		return d, errors.New("capture exceeds display dimensions")
+	}
+	var damageError C.int
+	if C.XDamageQueryExtension(d.display, &d.damageEvent, &damageError) == 0 {
+		return d, errors.New("XDamage unavailable")
+	}
+	if C.XShmQueryExtension(d.display) == 0 {
+		return d, errors.New("XShm unavailable")
+	}
+	d.damage = C.XDamageCreate(d.display, d.root, C.XDamageReportNonEmpty)
+	if partial {
+		d.region = C.XFixesCreateRegion(d.display, nil, 0)
+	}
+	// XShm retains this address, so the descriptor lives in native memory.
+	d.shm = (*C.XShmSegmentInfo)(C.calloc(1, C.sizeof_XShmSegmentInfo))
+	if d.shm == nil {
+		return d, errors.New("cannot allocate shared memory descriptor")
+	}
+	d.shm.shmid = -1
+	d.image = C.XShmCreateImage(d.display, d.attributes.visual, C.uint(d.attributes.depth), C.ZPixmap, nil, d.shm, C.uint(o.Width), C.uint(o.Height))
+	if d.image == nil {
+		return d, errors.New("cannot allocate capture image")
+	}
+	im := d.image
+	if im.bits_per_pixel != 32 || im.byte_order != C.LSBFirst || im.red_mask != 0xff0000 || im.green_mask != 0xff00 || im.blue_mask != 0xff {
+		return d, errors.New("unsupported screen pixel format")
+	}
+	d.shm.shmid = C.shmget(C.IPC_PRIVATE, C.size_t(int(im.bytes_per_line)*o.Height), C.IPC_CREAT|0600)
+	if d.shm.shmid < 0 {
+		return d, errors.New("cannot allocate shared memory")
+	}
+	address := C.shmat(d.shm.shmid, nil, 0)
+	if uintptr(address) == ^uintptr(0) {
+		return d, errors.New("cannot map shared memory")
+	}
+	d.shm.shmaddr = (*C.char)(address)
+	im.data = d.shm.shmaddr
+	if C.XShmAttach(d.display, d.shm) == 0 {
+		return d, errors.New("cannot attach shared memory")
+	}
+	C.XSync(d.display, 0)
+	if C.capture_error() != 0 {
+		return d, errors.New("X server rejected shared memory")
+	}
+	d.attached = true
+	// The kernel removes the segment even after a forced process exit.
+	if C.shmctl(d.shm.shmid, C.IPC_RMID, nil) != 0 {
+		return d, errors.New("cannot mark shared memory for removal")
+	}
+	return d, nil
+}
+
+func (d *desktop) close() {
+	if d.attached {
+		C.XShmDetach(d.display, d.shm)
+		C.XSync(d.display, 0)
+	}
+	if d.image != nil {
+		d.image.data = nil
+		C.XDestroyImage(d.image)
+	}
+	if d.shm != nil {
+		if d.shm.shmaddr != nil {
+			C.shmdt(unsafe.Pointer(d.shm.shmaddr))
+		}
+		if d.shm.shmid >= 0 {
+			C.shmctl(d.shm.shmid, C.IPC_RMID, nil)
+		}
+		C.free(unsafe.Pointer(d.shm))
+	}
+	if d.region != 0 {
+		C.XFixesDestroyRegion(d.display, d.region)
+	}
+	if d.damage != 0 {
+		C.XDamageDestroy(d.display, d.damage)
+	}
+	if d.display != nil {
+		C.XCloseDisplay(d.display)
+	}
+}
+
+func (d *desktop) changed() (bool, error) {
+	changed := false
+	var event C.XEvent
+	for C.XPending(d.display) != 0 {
+		C.XNextEvent(d.display, &event)
+		if C.capture_event_type(&event) == d.damageEvent+C.XDamageNotify {
+			changed = true
+			if d.region == 0 {
+				C.XDamageSubtract(d.display, d.damage, 0, 0)
+			}
+		}
+	}
+	if C.capture_error() != 0 {
+		return false, errors.New("X server capture error")
+	}
+	return changed, nil
+}
+
+func (d *desktop) wait(micros int64) error {
+	C.XFlush(d.display)
+	fd := C.struct_pollfd{fd: C.XConnectionNumber(d.display), events: C.POLLIN}
+	timeout := C.struct_timespec{tv_sec: C.time_t(micros / 1000000), tv_nsec: C.long(micros % 1000000 * 1000)}
+	ready, err := C.ppoll(&fd, 1, &timeout, nil)
+	if ready < 0 && err != syscall.EINTR {
+		return fmt.Errorf("display poll failed: %w", err)
+	}
+	if fd.revents&(C.POLLERR|C.POLLHUP|C.POLLNVAL) != 0 {
+		return errors.New("display disconnected")
+	}
+	return nil
+}
+
+func (d *desktop) capture(first bool) (int, int, error) {
+	x, y, right, bottom := 0, 0, d.width, d.height
+	if d.region != 0 {
+		C.XDamageSubtract(d.display, d.damage, 0, d.region)
+		C.XFixesRegionExtents(d.display, d.region, d.region)
+		var count C.int
+		rectangles := C.XFixesFetchRegion(d.display, d.region, &count)
+		if !first && count == 1 && rectangles != nil {
+			x = max(0, int(rectangles.x)) &^ 1
+			y = max(0, int(rectangles.y)) &^ 1
+			right = min(d.width, (int(rectangles.x)+int(rectangles.width)+1)&^1)
+			bottom = min(d.height, (int(rectangles.y)+int(rectangles.height)+1)&^1)
+		}
+		C.XFree(unsafe.Pointer(rectangles))
+		if x >= right || y >= bottom {
+			x, y, right, bottom = 0, 0, d.width, d.height
+		}
+		if int(d.image.width) != right-x || int(d.image.height) != bottom-y {
+			next := C.XShmCreateImage(d.display, d.attributes.visual, C.uint(d.attributes.depth), C.ZPixmap, d.shm.shmaddr, d.shm, C.uint(right-x), C.uint(bottom-y))
+			if next == nil {
+				return 0, 0, errors.New("cannot allocate capture region")
+			}
+			d.image.data = nil
+			C.XDestroyImage(d.image)
+			d.image = next
+		}
+	}
+	if C.XShmGetImage(d.display, d.root, d.image, C.int(x), C.int(y), ^C.ulong(0)) == 0 {
+		return 0, 0, errors.New("screen capture failed")
+	}
+	return x, y, nil
+}
+
+type dictionary struct{ pointer *C.AVDictionary }
+
+func (d *dictionary) set(key, value string) error {
+	k, v := C.CString(key), C.CString(value)
+	defer C.free(unsafe.Pointer(k))
+	defer C.free(unsafe.Pointer(v))
+	return avCheck(C.av_dict_set(&d.pointer, k, v, 0))
+}
+func (d *dictionary) close() { C.av_dict_free(&d.pointer) }
+
+type encoder struct {
+	codec         *C.AVCodecContext
+	mux           *C.AVFormatContext
+	stream        *C.AVStream
+	frame, input  *C.AVFrame
+	packet        *C.AVPacket
+	scale         *C.struct_SwsContext
+	matrix        colorMatrix
+	headerWritten bool
+}
+
+func newEncoder(o Options) (e *encoder, err error) {
+	e = &encoder{}
+	defer func() {
+		if err != nil {
+			e.close()
+		}
+	}()
+	if hasLibyuv {
+		e.matrix, err = newColorMatrix(o.BT709 != 0)
+		if err != nil {
+			return e, err
+		}
+	}
+	name := C.CString("libx264")
+	defer C.free(unsafe.Pointer(name))
+	implementation := C.avcodec_find_encoder_by_name(name)
+	if implementation == nil {
+		return e, errors.New("libx264 unavailable")
+	}
+	e.codec = C.avcodec_alloc_context3(implementation)
+	if e.codec == nil {
+		return e, errors.New("cannot allocate encoder")
+	}
+	c := e.codec
+	c.width, c.height = C.int(o.Width), C.int(o.Height)
+	c.pix_fmt = C.AV_PIX_FMT_YUV420P
+	if o.Profile == "high444" {
+		c.pix_fmt = C.AV_PIX_FMT_YUV444P
+	}
+	c.time_base = C.AVRational{num: 1, den: 90000}
+	c.framerate = C.AVRational{num: C.int(o.FPS), den: 1}
+	c.gop_size = C.int(max(1, (o.FPS*o.KeyframeMS+500)/1000))
+	c.max_b_frames, c.thread_count = 0, C.int(o.Threads)
+	c.rc_max_rate = C.int64_t(o.Kbps) * 1000
+	c.rc_buffer_size = C.int(o.Kbps * min(o.KeyframeMS, 100))
+	c.flags |= C.AV_CODEC_FLAG_LOW_DELAY
+	if o.BT709 != 0 {
+		c.color_primaries = C.AVCOL_PRI_BT709
+		c.color_trc = C.AVCOL_TRC_BT709
+		c.colorspace = C.AVCOL_SPC_BT709
+		c.color_range = C.AVCOL_RANGE_MPEG
+	}
+	dict := dictionary{}
+	defer dict.close()
+	parameters := fmt.Sprintf("fps=%d/1:repeat-headers=1:sliced-threads=1:slices=4:keyint=%d:min-keyint=%d:scenecut=0:chroma-qp-offset=%d:intra-refresh=%d", o.FPS, c.gop_size, c.gop_size, o.ChromaQP, o.IntraRefresh)
+	if o.Preset == "superfast" {
+		parameters += ":subme=0"
+	}
+	for _, option := range [][2]string{{"preset", o.Preset}, {"tune", "zerolatency"}, {"crf", strconv.Itoa(o.CRF)}, {"profile", o.Profile}, {"level", "4.2"}, {"forced-idr", "1"}, {"x264-params", parameters}} {
+		if err = dict.set(option[0], option[1]); err != nil {
+			return e, err
+		}
+	}
+	if err = avCheck(C.avcodec_open2(c, implementation, &dict.pointer)); err != nil {
+		return e, err
+	}
+	dict.close()
+	format, url := C.CString("rtp"), C.CString(o.URL)
+	defer C.free(unsafe.Pointer(format))
+	defer C.free(unsafe.Pointer(url))
+	if err = avCheck(C.avformat_alloc_output_context2(&e.mux, nil, format, url)); err != nil {
+		return e, err
+	}
+	e.stream = C.avformat_new_stream(e.mux, nil)
+	if e.stream == nil {
+		return e, errors.New("cannot allocate RTP stream")
+	}
+	e.stream.time_base = c.time_base
+	if err = avCheck(C.avcodec_parameters_from_context(e.stream.codecpar, c)); err != nil {
+		return e, err
+	}
+	if err = dict.set("payload_type", strconv.Itoa(o.Payload)); err != nil {
+		return e, err
+	}
+	if err = dict.set("ssrc", o.SSRC); err != nil {
+		return e, err
+	}
+	e.mux.flags |= C.AVFMT_FLAG_FLUSH_PACKETS
+	if err = avCheck(C.avio_open(&e.mux.pb, url, C.AVIO_FLAG_WRITE)); err != nil {
+		return e, err
+	}
+	if err = avCheck(C.avformat_write_header(e.mux, &dict.pointer)); err != nil {
+		return e, err
+	}
+	e.headerWritten = true
+	if !hasLibyuv {
+		e.scale = C.sws_alloc_context()
+		if e.scale == nil {
+			return e, errors.New("cannot allocate scaler")
+		}
+		for _, option := range []struct {
+			key   string
+			value int
+		}{{"srcw", o.Width}, {"srch", o.Height}, {"src_format", C.AV_PIX_FMT_BGR0}, {"dstw", o.Width}, {"dsth", o.Height}, {"dst_format", int(c.pix_fmt)}, {"sws_flags", C.SWS_LANCZOS}, {"threads", o.Threads}} {
+			key := C.CString(option.key)
+			err = avCheck(C.av_opt_set_int(unsafe.Pointer(e.scale), key, C.int64_t(option.value), 0))
+			C.free(unsafe.Pointer(key))
+			if err != nil {
+				return e, err
+			}
+		}
+		if err = avCheck(C.sws_init_context(e.scale, nil, nil)); err != nil {
+			return e, err
+		}
+		if o.BT709 != 0 {
+			coefficients := C.sws_getCoefficients(C.SWS_CS_ITU709)
+			if err = avCheck(C.sws_setColorspaceDetails(e.scale, coefficients, 0, coefficients, 0, 0, 1<<16, 1<<16)); err != nil {
+				return e, err
+			}
+		}
+		e.input = C.av_frame_alloc()
+		if e.input == nil {
+			return e, errors.New("cannot allocate input frame")
+		}
+	}
+	e.frame, e.packet = C.av_frame_alloc(), C.av_packet_alloc()
+	if e.frame == nil || e.packet == nil {
+		return e, errors.New("cannot allocate frame or packet")
+	}
+	e.frame.format, e.frame.width, e.frame.height = C.int(c.pix_fmt), c.width, c.height
+	e.frame.color_primaries, e.frame.color_trc = c.color_primaries, c.color_trc
+	e.frame.colorspace, e.frame.color_range = c.colorspace, c.color_range
+	if err = avCheck(C.av_frame_get_buffer(e.frame, 32)); err != nil {
+		return e, err
+	}
+	return e, nil
+}
+
+func (e *encoder) close() {
+	if e.headerWritten {
+		C.av_write_trailer(e.mux)
+	}
+	C.av_packet_free(&e.packet)
+	C.av_frame_free(&e.frame)
+	C.av_frame_free(&e.input)
+	C.avcodec_free_context(&e.codec)
+	if e.mux != nil {
+		if e.mux.pb != nil {
+			C.avio_closep(&e.mux.pb)
+		}
+		C.avformat_free_context(e.mux)
+	}
+	C.sws_freeContext(e.scale)
+	e.matrix.close()
+}
+
+func (e *encoder) convert(d *desktop, x, y int) error {
+	im := d.image
+	if e.scale != nil {
+		e.input.format, e.input.width, e.input.height = C.AV_PIX_FMT_BGR0, C.int(d.width), C.int(d.height)
+		e.input.buf[0] = C.capture_reference((*C.uint8_t)(unsafe.Pointer(im.data)), C.size_t(int(im.bytes_per_line)*d.height))
+		if e.input.buf[0] == nil {
+			return errors.New("cannot reference shared memory")
+		}
+		e.input.data[0], e.input.linesize[0] = (*C.uint8_t)(unsafe.Pointer(im.data)), im.bytes_per_line
+		err := avCheck(C.sws_scale_frame(e.scale, e.frame, e.input))
+		C.av_frame_unref(e.input)
+		return err
+	}
+	return e.matrix.convert(im, e.frame, x, y)
+}
+
+func (e *encoder) send(pts int64, keyframe bool) error {
+	e.frame.pts, e.frame.pict_type = C.int64_t(pts), C.AV_PICTURE_TYPE_NONE
+	if keyframe {
+		e.frame.pict_type = C.AV_PICTURE_TYPE_I
+	}
+	if err := avCheck(C.avcodec_send_frame(e.codec, e.frame)); err != nil {
+		return err
+	}
+	for {
+		code := C.avcodec_receive_packet(e.codec, e.packet)
+		if code == -C.EAGAIN || code == C.CAPTURE_AVERROR_EOF {
+			return nil
+		}
+		if err := avCheck(code); err != nil {
+			return err
+		}
+		C.av_packet_rescale_ts(e.packet, e.codec.time_base, e.stream.time_base)
+		e.packet.stream_index = e.stream.index
+		err := avCheck(C.av_interleaved_write_frame(e.mux, e.packet))
+		C.av_packet_unref(e.packet)
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// Open and every method on the stream must run on the same goroutine.
+// Xlib state is confined to its locked OS thread until Close.
+type Stream struct {
+	desktop *desktop
+	encoder *encoder
+}
+
+func Open(o Options) (*Stream, error) {
+	runtime.LockOSThread()
+	C.av_log_set_level(C.AV_LOG_WARNING)
+	d, err := newDesktop(o, hasLibyuv)
+	if err != nil {
+		runtime.UnlockOSThread()
+		return nil, err
+	}
+	e, err := newEncoder(o)
+	if err != nil {
+		d.close()
+		runtime.UnlockOSThread()
+		return nil, err
+	}
+	return &Stream{desktop: d, encoder: e}, nil
+}
+
+func (s *Stream) Close() {
+	if s.desktop == nil {
+		return
+	}
+	s.encoder.close()
+	s.desktop.close()
+	s.encoder, s.desktop = nil, nil
+	runtime.UnlockOSThread()
+}
+
+func NowMicros() int64                                { return int64(C.av_gettime_relative()) }
+func (s *Stream) Changed() (bool, error)              { return s.desktop.changed() }
+func (s *Stream) Wait(micros int64) error             { return s.desktop.wait(micros) }
+func (s *Stream) Send(pts int64, keyframe bool) error { return s.encoder.send(pts, keyframe) }
+
+func (s *Stream) Capture(first bool) (copied, converted int64, err error) {
+	if err = avCheck(C.av_frame_make_writable(s.encoder.frame)); err != nil {
+		return
+	}
+	x, y, err := s.desktop.capture(first)
+	if err != nil {
+		return
+	}
+	copied = NowMicros()
+	err = s.encoder.convert(s.desktop, x, y)
+	converted = NowMicros()
+	return
+}

package/native/internal/capture/native.h

@@ -0,0 +1,31 @@
+#define _GNU_SOURCE
+#define XUTIL_DEFINE_FUNCTIONS
+#include <X11/Xlib.h>
+#include <X11/Xutil.h>
+#include <X11/extensions/XShm.h>
+#include <X11/extensions/Xdamage.h>
+#include <X11/extensions/Xfixes.h>
+#include <errno.h>
+#include <poll.h>
+#include <stdlib.h>
+#include <sys/shm.h>
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/opt.h>
+#include <libavutil/time.h>
+#include <libswscale/swscale.h>
+
+static int capture_x_error;
+static int capture_handle_error(Display *display, XErrorEvent *event) {
+    capture_x_error = event->error_code;
+    return 0;
+}
+static void capture_install_handler(void) { capture_x_error = 0; XSetErrorHandler(capture_handle_error); }
+static int capture_error(void) { return capture_x_error; }
+static int capture_event_type(XEvent *event) { return event->type; }
+// XShm owns the pixels; swscale only borrows them synchronously.
+static void capture_keep_shm(void *opaque, uint8_t *data) {}
+static AVBufferRef *capture_reference(uint8_t *data, size_t size) {
+    return av_buffer_create(data, size, capture_keep_shm, NULL, AV_BUFFER_FLAG_READONLY);
+}
+enum { CAPTURE_AVERROR_EOF = AVERROR_EOF };