replicas-computer 0.3.7 → 0.3.8

package/native/internal/capture/native_test.go

@@ -0,0 +1,22 @@
+//go:build libyuv && capturetest
+
+package capture
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestPartialCapture(t *testing.T) {
+	for _, size := range [][2]int{{128, 96}, {94, 66}} {
+		for _, i420 := range []bool{true, false} {
+			for _, bt709 := range []bool{true, false} {
+				t.Run(fmt.Sprintf("%dx%d/i420=%t/bt709=%t", size[0], size[1], i420, bt709), func(t *testing.T) {
+					if err := checkPartialCapture(size[0], size[1], i420, bt709); err != nil {
+						t.Fatal(err)
+					}
+				})
+			}
+		}
+	}
+}

package/native/internal/capture/options.go

@@ -0,0 +1,7 @@
+package capture
+
+type Options struct {
+	Width, Height, FPS, Threads, CRF, Kbps                     int
+	KeyframeMS, IntraRefresh, ChromaQP, Warmup, Payload, BT709 int
+	Preset, Profile, URL, SSRC                                 string
+}

package/native/internal/capture/yuv.go

@@ -0,0 +1,55 @@
+//go:build libyuv
+
+package capture
+
+/*
+#include <stdlib.h>
+#include <X11/Xlib.h>
+#include <libavutil/frame.h>
+#include <libyuv/convert.h>
+#include <libyuv/convert_from_argb.h>
+#include <libyuv/row.h>
+*/
+import "C"
+
+import (
+	"errors"
+	"unsafe"
+)
+
+const hasLibyuv = true
+
+type colorMatrix struct{ pointer *C.struct_ArgbConstants }
+
+func newColorMatrix(bt709 bool) (colorMatrix, error) {
+	var pointer unsafe.Pointer
+	if C.posix_memalign(&pointer, 64, C.sizeof_struct_ArgbConstants) != 0 {
+		return colorMatrix{}, errors.New("cannot allocate color matrix")
+	}
+	m := colorMatrix{(*C.struct_ArgbConstants)(pointer)}
+	*m.pointer = C.kArgbI601Constants
+	if bt709 {
+		*m.pointer = C.kArgbH709Constants
+	}
+	for i := range m.pointer.kAddUV {
+		m.pointer.kAddUV[i] += 128
+	}
+	return m, nil
+}
+
+func (m colorMatrix) close() { C.free(unsafe.Pointer(m.pointer)) }
+
+func (m colorMatrix) convert(im *C.XImage, f *C.AVFrame, x, y int) error {
+	chroma := 0
+	if f.format == C.AV_PIX_FMT_YUV420P {
+		chroma = 1
+	}
+	// The clipped, aligned region stays inside these native-owned image planes.
+	py := (*C.uint8_t)(unsafe.Add(unsafe.Pointer(f.data[0]), y*int(f.linesize[0])+x))
+	pu := (*C.uint8_t)(unsafe.Add(unsafe.Pointer(f.data[1]), (y>>chroma)*int(f.linesize[1])+(x>>chroma)))
+	pv := (*C.uint8_t)(unsafe.Add(unsafe.Pointer(f.data[2]), (y>>chroma)*int(f.linesize[2])+(x>>chroma)))
+	if chroma != 0 {
+		return avCheck(C.ARGBToI420Matrix((*C.uint8_t)(unsafe.Pointer(im.data)), im.bytes_per_line, py, f.linesize[0], pu, f.linesize[1], pv, f.linesize[2], m.pointer, im.width, im.height))
+	}
+	return avCheck(C.ARGBToI444Matrix((*C.uint8_t)(unsafe.Pointer(im.data)), im.bytes_per_line, py, f.linesize[0], pu, f.linesize[1], pv, f.linesize[2], m.pointer, im.width, im.height))
+}

package/native/internal/capture/yuv_fallback.go

@@ -0,0 +1,23 @@
+//go:build !libyuv
+
+package capture
+
+/*
+#include <X11/Xlib.h>
+#include <libavutil/frame.h>
+*/
+import "C"
+
+import "errors"
+
+const hasLibyuv = false
+
+type colorMatrix struct{}
+
+func newColorMatrix(bool) (colorMatrix, error) {
+	return colorMatrix{}, errors.New("libyuv unavailable")
+}
+func (colorMatrix) close() {}
+func (colorMatrix) convert(*C.XImage, *C.AVFrame, int, int) error {
+	return errors.New("libyuv unavailable")
+}

package/native/main.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/replicas-group/replicas/computer-use/native/internal/capture"
+)
+
+func run(o capture.Options) error {
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
+	defer signal.Stop(stop)
+	stream, err := capture.Open(o)
+	if err != nil {
+		return err
+	}
+	defer stream.Close()
+	started := capture.NowMicros()
+	last, lastKeyframe, reportAt := started-1000000, started-1000000, started
+	interval := int64((1000000 + o.FPS - 1) / o.FPS)
+	busyUS, captureUS, scaleUS, encodeUS, previousPTS := int64(0), int64(0), int64(0), int64(0), int64(-1)
+	dirty := true
+	number, changedFrames, sentFrames := 0, 0, 0
+	for {
+		select {
+		case <-stop:
+			return nil
+		default:
+		}
+		changed, err := stream.Changed()
+		if err != nil {
+			return err
+		}
+		dirty = dirty || changed
+		now := capture.NowMicros()
+		idleDue := last + max(interval, 200000)
+		if o.IntraRefresh == 0 {
+			idleDue = min(idleDue, lastKeyframe+int64(o.KeyframeMS)*1000)
+		}
+		due := max(last+interval, idleDue)
+		if dirty || number <= o.Warmup || number < 3 {
+			due = last + interval
+		}
+		if now < due {
+			if err = stream.Wait(due - now); err != nil {
+				return err
+			}
+			continue
+		}
+		copied, converted := now, now
+		if dirty {
+			copied, converted, err = stream.Capture(number == 0)
+			if err != nil {
+				return err
+			}
+			changedFrames++
+			dirty = false
+		}
+		pts := max(previousPTS+1, (now-started)*90/1000)
+		previousPTS = pts
+		keyframe := number == 0 || (o.Warmup > 0 && number == o.Warmup) || (o.IntraRefresh == 0 && now-lastKeyframe >= int64(o.KeyframeMS)*1000)
+		if keyframe {
+			lastKeyframe = now
+		}
+		if err = stream.Send(pts, keyframe); err != nil {
+			return err
+		}
+		ended := capture.NowMicros()
+		captureUS += copied - now
+		scaleUS += converted - copied
+		encodeUS += ended - converted
+		busyUS += ended - now
+		sentFrames++
+		number++
+		if now-last < interval*2 {
+			last += interval
+		} else {
+			last = now
+		}
+		if ended-reportAt >= 1000000 {
+			_, err = fmt.Fprintf(os.Stdout, "{\"elapsedMs\":%.3f,\"busyMs\":%.3f,\"captureMs\":%.3f,\"scaleMs\":%.3f,\"encodeMs\":%.3f,\"frames\":%d,\"changedFrames\":%d}\n", float64(ended-reportAt)/1000, float64(busyUS)/1000, float64(captureUS)/1000, float64(scaleUS)/1000, float64(encodeUS)/1000, sentFrames, changedFrames)
+			if err != nil {
+				return err
+			}
+			reportAt = ended
+			busyUS, captureUS, scaleUS, encodeUS = 0, 0, 0, 0
+			changedFrames, sentFrames = 0, 0
+		}
+	}
+}
+
+func main() {
+	o, err := parseOptions()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "capture:", err)
+		os.Exit(2)
+	}
+	if err = run(o); err != nil {
+		fmt.Fprintln(os.Stderr, "capture:", err)
+		os.Exit(1)
+	}
+}

package/native/options.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/replicas-group/replicas/computer-use/native/internal/capture"
+)
+
+func parseOptions() (capture.Options, error) {
+	o := capture.Options{}
+	f := flag.NewFlagSet("capture", flag.ContinueOnError)
+	numbers := []struct {
+		name               string
+		value              *int
+		initial, low, high int
+	}{
+		{"width", &o.Width, 0, 2, 8192}, {"height", &o.Height, 0, 2, 8192},
+		{"fps", &o.FPS, 60, 1, 240}, {"threads", &o.Threads, 4, 1, 16},
+		{"crf", &o.CRF, 14, 0, 51}, {"kbps", &o.Kbps, 20000, 1, 1000000},
+		{"keyframe-ms", &o.KeyframeMS, 1000, 1, 10000}, {"intra-refresh", &o.IntraRefresh, 1, 0, 1},
+		{"chroma-qp", &o.ChromaQP, 0, -12, 12}, {"warmup", &o.Warmup, 0, 0, 60},
+		{"payload", &o.Payload, 96, 0, 127}, {"bt709", &o.BT709, 1, 0, 1},
+	}
+	for _, n := range numbers {
+		f.IntVar(n.value, n.name, n.initial, "")
+	}
+	f.StringVar(&o.Preset, "preset", "superfast", "")
+	f.StringVar(&o.Profile, "profile", "main", "")
+	f.StringVar(&o.URL, "url", "", "")
+	f.StringVar(&o.SSRC, "ssrc", "", "")
+	if err := f.Parse(os.Args[1:]); err != nil {
+		return o, err
+	}
+	var invalid error
+	f.Visit(func(f *flag.Flag) {
+		for _, n := range numbers {
+			if n.name == f.Name && (*n.value < n.low || *n.value > n.high) {
+				invalid = fmt.Errorf("invalid --%s", n.name)
+			}
+		}
+	})
+	if invalid != nil {
+		return o, invalid
+	}
+	if f.NArg() != 0 || o.Width == 0 || o.Height == 0 || o.URL == "" || o.SSRC == "" || o.Width%2 != 0 || o.Height%2 != 0 {
+		return o, errors.New("width, height, url and ssrc are required; dimensions must be even")
+	}
+	return o, nil
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.3.7",
+  "version": "0.3.8",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",
@@ -13,14 +13,21 @@
     "scripts/build-webrtc.sh",
     "patches/libdatachannel-dtls-bio.patch",
     "patches/libdatachannel-nack-duplicates.patch",
-    "patches/libjuice-ice-role-presence.patch"
+    "patches/libjuice-ice-role-presence.patch",
+    "scripts/build-capture.sh",
+    "patches/libyuv-chroma-rounding.patch",
+    "native/go.mod",
+    "native/*.go",
+    "native/internal"
   ],
   "scripts": {
     "build": "bun run typecheck && tsup",
     "typecheck": "tsc --noEmit",
     "dev": "bun run src/index.ts",
     "dev-pack": "bun run build && npm pack --pack-destination /tmp",
-    "prepublishOnly": "bun run build"
+    "prepublishOnly": "bun run build",
+    "build:capture": "bash scripts/build-capture.sh",
+    "test:capture": "bash scripts/build-capture.sh test"
   },
   "keywords": [
     "replicas",

package/patches/libyuv-chroma-rounding.patch

@@ -0,0 +1,83 @@
+diff --git a/source/row_gcc.cc b/source/row_gcc.cc
+index 90d6786..f52e61b 100644
+--- a/source/row_gcc.cc
++++ b/source/row_gcc.cc
+@@ -1969,8 +1969,7 @@ void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb,
+                                 int width,
+                                 const struct ArgbConstants* c) {
+   asm volatile(
+-      "pcmpeqb     %%xmm5,%%xmm5                 \n"  // 0x8000
+-      "psllw       $15,%%xmm5                    \n"
++      "movdqa      0x80(%4),%%xmm5               \n"
+       "movdqa      0x20(%4),%%xmm3               \n"  // kRGBToU
+       "movdqa      0x40(%4),%%xmm4               \n"  // kRGBToV
+       "sub         %1,%2                         \n"
+@@ -2042,8 +2041,7 @@ void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb,
+   asm volatile(
+       "vmovdqa     0x20(%4),%%ymm3               \n"  // kRGBToU
+       "vmovdqa     0x40(%4),%%ymm4               \n"  // kRGBToV
+-      "vpcmpeqb    %%ymm5,%%ymm5,%%ymm5          \n"  // 0x8000
+-      "vpsllw      $15,%%ymm5,%%ymm5             \n"
++      "vmovdqa     0x80(%4),%%ymm5               \n"
+       "vmovdqa     %5,%%ymm7                     \n"
+       "sub         %1,%2                         \n"
+ 
+@@ -2115,7 +2113,7 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
+       "vbroadcasti32x4 0x20(%4),%%zmm3           \n"  // kRGBToU
+       "vbroadcasti32x4 0x40(%4),%%zmm4           \n"  // kRGBToV
+       "vpternlogd  $0xff,%%zmm16,%%zmm16,%%zmm16 \n"  // -1
+-      "vpsllw      $15,%%zmm16,%%zmm5            \n"  // 0x8000
++      "vbroadcasti32x4 0x80(%4),%%zmm5           \n"
+       "vmovups     %5,%%zmm17                    \n"
+       "sub         %1,%2                         \n"
+ 
+@@ -2135,8 +2133,8 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
+       "vpmaddwd    %%zmm16,%%zmm18,%%zmm18       \n"
+       "vpackssdw   %%zmm1,%%zmm0,%%zmm0          \n"  // mutates
+       "vpackssdw   %%zmm18,%%zmm2,%%zmm2         \n"
+-      "vpsubw      %%zmm5,%%zmm0,%%zmm0          \n"
+-      "vpsubw      %%zmm5,%%zmm2,%%zmm2          \n"
++      "vpaddw      %%zmm5,%%zmm0,%%zmm0          \n"
++      "vpaddw      %%zmm5,%%zmm2,%%zmm2          \n"
+       "vpsrlw      $0x8,%%zmm0,%%zmm0            \n"
+       "vpsrlw      $0x8,%%zmm2,%%zmm2            \n"
+       "vpackuswb   %%zmm2,%%zmm0,%%zmm0          \n"  // mutates
+@@ -2157,8 +2155,8 @@ void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb,
+       "vpmaddwd    %%zmm16,%%zmm18,%%zmm18       \n"
+       "vpackssdw   %%zmm1,%%zmm0,%%zmm0          \n"  // mutates
+       "vpackssdw   %%zmm18,%%zmm2,%%zmm2         \n"
+-      "vpsubw      %%zmm5,%%zmm0,%%zmm0          \n"
+-      "vpsubw      %%zmm5,%%zmm2,%%zmm2          \n"
++      "vpaddw      %%zmm5,%%zmm0,%%zmm0          \n"
++      "vpaddw      %%zmm5,%%zmm2,%%zmm2          \n"
+       "vpsrlw      $0x8,%%zmm0,%%zmm0            \n"
+       "vpsrlw      $0x8,%%zmm2,%%zmm2            \n"
+       "vpackuswb   %%zmm2,%%zmm0,%%zmm0          \n"  // mutates
+@@ -2230,8 +2228,7 @@ void ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb,
+       "pavgw       %%xmm2,%%xmm1                 \n"
+       "packuswb    %%xmm1,%%xmm0                 \n"  // mutates
+ 
+-      "movdqa      %%xmm6,%%xmm2                 \n"
+-      "psllw       $15,%%xmm2                    \n"  // 0x8000
++      "movdqa      0x80(%5),%%xmm2               \n"
+       "movdqa      %%xmm0,%%xmm1                 \n"
+       "pmaddubsw   %%xmm5,%%xmm1                 \n"  // 4 V
+       "pmaddubsw   %%xmm4,%%xmm0                 \n"  // 4 U
+@@ -2305,7 +2302,7 @@ void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
+       "vpackuswb   %%ymm1,%%ymm0,%%ymm0          \n"  // mutates
+       "vpermq      $0xd8,%%ymm0,%%ymm0           \n"  // 8 ARGB Pixels
+ 
+-      "vpsllw      $15,%%ymm6,%%ymm2             \n"  // 0x8000
++      "vmovdqa     0x80(%5),%%ymm2               \n"
+       "vpmaddubsw  %%ymm5,%%ymm0,%%ymm1          \n"  // 8 V
+       "vpmaddubsw  %%ymm4,%%ymm0,%%ymm0          \n"  // 8 U
+       "vphaddw     %%ymm1,%%ymm0,%%ymm0          \n"  // uuuuvvvv uuuuvvvv
+@@ -2362,7 +2359,7 @@ void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb,
+       "vbroadcasti32x4 0x40(%5),%%zmm5           \n"  // RGBToV
+       "vpternlogd  $0xff,%%zmm16,%%zmm16,%%zmm16 \n"
+       "vpabsb      %%zmm16,%%zmm20               \n"  // 0x0101
+-      "vpsllw      $15,%%zmm16,%%zmm17           \n"  // 0x8000
++      "vbroadcasti32x4 0x80(%5),%%zmm17          \n"
+       "vbroadcasti32x4 %6,%%zmm21                \n"  // kShuffleAARRGGBB
+       "vmovups     %7,%%zmm18                    \n"  // kPermdARGBToY_AVX512BW
+       "vmovups     %8,%%zmm19                    \n"  // kPermdARGBToUV_AVX512BW

package/scripts/build-capture.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+package_dir="$(cd "$(dirname "$0")/.." && pwd)"
+mkdir -p "$package_dir/dist"
+export GOTOOLCHAIN=go1.27.1 CGO_ENABLED=1
+build_tags=()
+case "$("${CC:-cc}" -dumpmachine)" in
+  x86_64-*)
+    build_root="$(mktemp -d)"
+    trap 'rm -rf -- "$build_root"' EXIT
+    git init -q "$build_root/source"
+    git -C "$build_root/source" fetch --depth 1 https://chromium.googlesource.com/libyuv/libyuv 2dd4257364d39c38d79465c4ddc4b93137fe729b
+    git -C "$build_root/source" checkout -q --detach FETCH_HEAD
+    git -C "$build_root/source" apply "$package_dir/patches/libyuv-chroma-rounding.patch"
+    cmake -S "$build_root/source" -B "$build_root/build" -DCMAKE_BUILD_TYPE=Release -DLIBYUV_DISABLE_JPEG=ON -DUNIT_TEST=OFF
+    cmake --build "$build_root/build" --target yuv -j 2
+    export CGO_CFLAGS="${CGO_CFLAGS:--O2 -g} -I$build_root/source/include"
+    export CGO_LDFLAGS="${CGO_LDFLAGS:-} $build_root/build/libyuv.a -lstdc++ -lm"
+    build_tags=(-tags libyuv)
+    cp "$build_root/source/LICENSE" "$package_dir/dist/libyuv-LICENSE"
+    ;;
+esac
+cd "$package_dir/native"
+if [[ "${1:-}" == test ]]; then
+  [[ ${#build_tags[@]} -gt 0 ]] || { echo 'Partial capture tests require x86-64 libyuv' >&2; exit 1; }
+  GOEXPERIMENT=cgocheck2 xvfb-run -a -s '-screen 0 128x96x24 -nolisten tcp' go test -count=1 -tags libyuv,capturetest ./...
+  exit
+fi
+go build -mod=readonly -trimpath -ldflags='-s -w' "${build_tags[@]}" -o "$package_dir/dist/replicas-capture.new" .
+mv "$package_dir/dist/replicas-capture.new" "$package_dir/dist/replicas-capture"