replicas-computer 0.3.12 → 0.3.14

package/README.md

@@ -17,7 +17,7 @@ The versioned computer-use runtime for Replicas workspace images.
 - `xsettingsd` broadcasts theme changes to running GTK apps and Chrome. Chrome follows the dashboard theme without reopening tabs; launch it without a `GTK_THEME` override.
 - `assets/desktop/` owns the desktop theme: `brand-wallpaper.svg` (regenerate with `bun scripts/render-wallpaper.ts`), `tint2rc` dock, `openbox-theme/`, `gtk-settings.ini`, the `replicas-terminal` wrapper, `themes/` palettes that `start-services.sh` renders into those templates (`--theme light|dark` switches a running desktop), and the bundled fonts (static TTF instances of General Sans and Geist Mono; variable and WOFF2 builds render with wrong metrics in Pango and VTE).
 
-The viewer and agent command paths are independent. One on-demand encoder pair is shared by all viewers, stops when the last viewer disconnects, and restarts without reloading the page.
+The viewer and agent command paths are independent. One on-demand encoder pair is shared by all viewers, stops when the last viewer disconnects, and restarts without reloading the page. A viewer out of sight (the dashboard keeps one connected while the Desktop tab is hidden) reports `standby`, detected with an `IntersectionObserver` because a hidden iframe keeps its last layout size; while every viewer is in standby the encoder runs at 5 fps on the 1.5 Mbps rung, and the first viewer to become visible swaps it back to full rate without a gap.
 
 Chrome disables GPU compositing when no `/dev/dri/renderD*` device is available. This avoids Mesa's CPU-rendered compositor overhead before capture while retaining WebGL support. Desktops with a hardware render device keep Chrome's normal compositing path.
 

package/dist/index.js

@@ -2937,7 +2937,7 @@ async function computerStatusCommand() {
 }
 
 // src/version.ts
-var COMPUTER_USE_VERSION = "0.3.12";
+var COMPUTER_USE_VERSION = "0.3.14";
 var COMPUTER_USE_PROTOCOL_VERSION = 1;
 
 // src/program.ts
@@ -3628,6 +3628,8 @@ function validInput(value) {
       return true;
     case "theme":
       return value.theme === "light" || value.theme === "dark";
+    case "standby":
+      return typeof value.standby === "boolean";
     case "ping":
     case "pong":
       return finite(value.sentAt);
@@ -4069,7 +4071,7 @@ var WebRtcViewer = class {
 // src/video-encoder.ts
 var captureBinary = resolve2(dirname6(fileURLToPath(import.meta.url)), "..", "dist", "replicas-capture");
 function superviseVideoEncoder(options) {
-  const { display: display2, width: width2, height: height2, fps: fps2, colorMode: colorMode2 } = options;
+  const { display: display2, width: width2, height: height2, colorMode: colorMode2 } = options;
   const env = { ...process.env, DISPLAY: display2 };
   let nativeCapture = process.env.REPLICAS_DESKTOP_STREAM_CAPTURE !== "ffmpeg" && existsSync6(captureBinary);
   let captureMetrics;
@@ -4098,7 +4100,7 @@ function superviseVideoEncoder(options) {
     onPacket: options.onPacket,
     joinable: (packet) => isKeyframeStart(rtpPayload(packet)),
     args: (udpPort, warmupFrames) => {
-      const { quality, keyframeMs, profile: videoProfile, intraRefresh, pixelFormat, chromaQpOffset, preset } = options.params();
+      const { quality, keyframeMs, fps: fps2, profile: videoProfile, intraRefresh, pixelFormat, chromaQpOffset, preset } = options.params();
       const keyframeInterval = Math.max(1, Math.round(fps2 * keyframeMs / 1e3));
       if (nativeCapture) return [
         "--width",
@@ -4445,6 +4447,8 @@ var width = Number.parseInt(process.env.REPLICAS_DESKTOP_WIDTH || String(DESKTOP
 var height = Number.parseInt(process.env.REPLICAS_DESKTOP_HEIGHT || String(DESKTOP_VIEWER_HEIGHT), 10);
 var basePreset = availableParallelism2() >= 4 ? "superfast" : "ultrafast";
 var fps = Number.parseInt(process.env.REPLICAS_DESKTOP_STREAM_FPS || "", 10) || 60;
+var standbyFps = 5;
+var standbyRung = 5;
 var port = Number.parseInt(process.env.REPLICAS_DESKTOP_STREAM_PORT || String(DESKTOP_STREAM_PORT), 10);
 var display = process.env.REPLICAS_DESKTOP_DISPLAY?.replace(/^([^:])/, ":$1") || ":99";
 var webrtcEnabled = process.env.REPLICAS_DESKTOP_STREAM_WEBRTC !== "0";
@@ -4601,8 +4605,11 @@ async function runMediaServer() {
     if (accepted.length === 0) return "main";
     return PROFILE_ORDER.find((candidate) => (!requires420 || candidate !== "high444") && accepted.every((set) => set.has(candidate))) ?? "baseline";
   };
+  const allStandby = () => viewers.size > 0 && [...viewers.values()].every((viewer) => viewer.standby);
+  const streamFps = () => allStandby() ? Math.min(fps, standbyFps) : fps;
   const encoderParams = () => {
-    const quality = ladder()[rung];
+    const quality = ladder()[allStandby() ? Math.max(rung, standbyRung) : rung];
+    const encodeFps = streamFps();
     const sockets = socketViewers().length > 0;
     const keyframeMs = sockets ? Math.min(quality.keyframeMs, fallbackKeyframeMs) : quality.keyframeMs;
     const videoProfile = profile();
@@ -4615,8 +4622,9 @@ async function runMediaServer() {
       keyframeMs,
       intraRefresh,
       pixelFormat,
+      fps: encodeFps,
       chromaQpOffset: profiles[activeProfile].chromaQpOffset,
-      signature: `rung ${rung} \xB7 ${activeProfile} crf ${quality.crf} \xB7 ${preset} \xB7 ${videoProfile} \xB7 keyframe ${keyframeMs}ms${intraRefresh ? " refresh" : ""}`
+      signature: `rung ${rung} \xB7 ${activeProfile} crf ${quality.crf} \xB7 ${preset} \xB7 ${videoProfile} \xB7 keyframe ${keyframeMs}ms${intraRefresh ? " refresh" : ""} \xB7 ${encodeFps} fps`
     };
   };
   const budgetBps = () => ladder()[rung].kbps * 1e3 * budgetFactor;
@@ -4663,7 +4671,7 @@ async function runMediaServer() {
       }
     }
     const last = Boolean(packet[1] & 128);
-    videoSplicer.splice(packet, generation, Math.round(9e4 / fps));
+    videoSplicer.splice(packet, generation, Math.round(9e4 / streamFps()));
     let rtp;
     let sockets = false;
     for (const viewer of viewers.values()) {
@@ -4699,13 +4707,12 @@ async function runMediaServer() {
     display,
     width,
     height,
-    fps,
     colorMode,
     params: encoderParams,
     onPacket: onVideoPacket,
     record
   });
-  const expectedFps = () => videoEncoder.capture().captureBackend === "native" ? Math.max(1, Math.min(fps, encoderFps)) : fps;
+  const expectedFps = () => videoEncoder.capture().captureBackend === "native" ? Math.max(1, Math.min(streamFps(), encoderFps)) : streamFps();
   const audioEncoder = superviseEncoder({
     label: "audio",
     log: (message) => record("audio encoder", message),
@@ -4876,7 +4883,7 @@ async function runMediaServer() {
     if (viewers.size === 0) return;
     const { captureBackend, captureMetrics } = videoEncoder.capture();
     const captureBusy = captureBackend !== "native" || !captureMetrics || captureMetrics.busyMs / captureMetrics.elapsedMs > 0.8;
-    if (videoEncoder.running() && encoderFps < fps * 0.9 && captureBusy) {
+    if (videoEncoder.running() && encoderFps < streamFps() * 0.9 && captureBusy) {
       cleanTransportSeconds = 0;
       congestedSeconds = 0;
       comfortableSeconds = 0;
@@ -5143,6 +5150,12 @@ async function runMediaServer() {
       case "theme":
         applyDesktopTheme(message.theme);
         return;
+      case "standby":
+        if (viewer.standby === message.standby) return;
+        viewer.standby = message.standby;
+        record(message.standby ? "viewer standby" : "viewer active", `${[...viewers.values()].filter((v) => v.standby).length} of ${viewers.size} in standby`);
+        syncEncoder();
+        return;
       default:
         viewer.input.handle(message);
     }
@@ -5185,6 +5198,8 @@ async function runMediaServer() {
         audioBytes,
         videoKbps,
         encoderFps,
+        streamFps: streamFps(),
+        standbyViewers: all.filter((viewer) => viewer.standby).length,
         ...videoEncoder.capture(),
         cpu,
         clientStats: all.flatMap((viewer) => viewer.stats ? [viewer.stats] : []),
@@ -5238,13 +5253,14 @@ async function runMediaServer() {
       socket.destroy();
       return;
     }
-    webSocketServer.handleUpgrade(request, socket, head, (client) => webSocketServer.emit("connection", client, parseStreamRequest(url.searchParams, defaultRequest)));
+    webSocketServer.handleUpgrade(request, socket, head, (client) => webSocketServer.emit("connection", client, parseStreamRequest(url.searchParams, defaultRequest), url.searchParams.get("standby") === "1"));
   });
-  webSocketServer.on("connection", (socket, request) => {
+  webSocketServer.on("connection", (socket, request, standby) => {
     const viewer = {
       socket,
       input: new InputBridge(display, width, height),
       request,
+      standby,
       fallback: false,
       waitingForKeyframe: true,
       withheldFrames: 0,
@@ -5269,7 +5285,7 @@ async function runMediaServer() {
       if (!adaptive) rung = 0;
     }
     viewers.set(socket, viewer);
-    record("viewer connected", `${request.transport} requested, ${viewers.size} connected`);
+    record("viewer connected", `${request.transport} requested${standby ? " in standby" : ""}, ${viewers.size} connected`);
     const reply = (message) => {
       if (socket.readyState === WebSocket2.OPEN) socket.send(JSON.stringify(message));
     };

package/dist/stream-viewer.js

@@ -45,6 +45,7 @@ var DESKTOP_WEBRTC_FAILURE_REASONS = {
   video_stalled: "WebRTC video stopped arriving",
   full_chroma_undecodable: "WebRTC could not decode H.264 High 4:4:4"
 };
+var DESKTOP_QUALITY_MESSAGE = "replicas-desktop-quality";
 
 // src/input-validation.ts
 var finite = (value) => typeof value === "number" && Number.isFinite(value);
@@ -100,6 +101,7 @@ diagnostics.classList.toggle("visible", showDiagnostics);
 var webrtcFrameTimeoutMs = 6e3;
 var webrtcSkipsAfterFailure = 3;
 var pointerIntervalMs = 4;
+var qualityWindowMs = 3e4;
 var fullChromaFailureKey = "replicas-desktop-4:4:4-failed";
 var skipFullChroma = false;
 var fullChromaFailure;
@@ -108,6 +110,8 @@ try {
 } catch {
 }
 var socket;
+var standby = innerWidth === 0 || innerHeight === 0;
+var standbyTimer;
 var config;
 var transport;
 var transportReason = "";
@@ -151,6 +155,18 @@ var inboundAt = 0;
 var inboundHistory = [];
 var telemetry;
 var route = { path: "", codec: "", decoded: "", kbps: 0, displayedFps: 0 };
+var qualitySamples = [];
+var qualityWindow;
+var qualityUnseen = { decoded: 0, presented: 0, samples: 0 };
+var qualitySequence = 0;
+var presentedAt = 0;
+var socketDecodedFrames = 0;
+var restartQuality = () => {
+  qualitySamples = [];
+  qualityWindow = void 0;
+  qualitySequence = 0;
+  presentedAt = 0;
+};
 var state = {};
 globalThis.__replicasDesktop = state;
 var viewerEvents = [];
@@ -290,6 +306,7 @@ var configureVideo = async (next, preferHardware = true) => {
   activeVideoConfig = selectedConfig;
   const decoder = new VideoDecoder({
     output: (frame) => {
+      socketDecodedFrames += 1;
       const writer = videoWriter;
       if (!writer || (writer.desiredSize ?? 1) <= 0) {
         frame.close();
@@ -443,6 +460,7 @@ var useFallback = (reasonCode, failed = false) => {
   if (failed) webrtcSkips = webrtcSkipsAfterFailure;
   note("fallback to socket", reason);
   transport = "websocket";
+  restartQuality();
   transportReason = reason;
   route = { ...route, path: "proxy", codec: config.videoCodec };
   announceTransport();
@@ -624,11 +642,13 @@ var connect = () => {
   candidateErrors = 0;
   negotiationErrorName = "";
   lastFrameAt = 0;
+  restartQuality();
   route = { path: "", codec: "", decoded: "", kbps: 0, displayedFps: 0 };
   announceTransport();
   socket?.close();
   showStatus(reconnectAttempt ? "reconnecting\u2026" : "connecting\u2026", true);
   const query = new URLSearchParams([...params].filter(([key]) => ["transport", "profile", "adapt", "ice"].includes(key)));
+  if (standby) query.set("standby", "1");
   const nextSocket = new WebSocket(`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/stream?${query}`);
   nextSocket.binaryType = "arraybuffer";
   socket = nextSocket;
@@ -685,6 +705,8 @@ var observeFrame = () => {
     presentedFrames += lastPresentedCount ? Math.max(0, metadata.presentedFrames - lastPresentedCount) : 1;
     lastPresentedCount = metadata.presentedFrames;
     lastFrameAt = performance.now();
+    if (presentedAt && qualityWindow) qualityWindow.longestGapMs = Math.max(qualityWindow.longestGapMs, lastFrameAt - Math.max(presentedAt, qualityWindow.at));
+    presentedAt = lastFrameAt;
     status.classList.remove("visible");
     observeFrame();
   });
@@ -730,6 +752,21 @@ window.addEventListener("keyup", (event) => {
 }, { capture: true });
 window.addEventListener("blur", () => send({ type: "release" }));
 window.addEventListener("resize", fitScreen);
+var setStandby = (next) => {
+  clearTimeout(standbyTimer);
+  if (next === standby) return;
+  standby = next;
+  note(standby ? "standby" : "active");
+  sendSocket({ type: "standby", standby });
+  if (!standby && !audioPrompt.hidden) unlockAudio();
+};
+new IntersectionObserver(([entry]) => {
+  if (entry.isIntersecting) setStandby(false);
+  else if (!standby) {
+    clearTimeout(standbyTimer);
+    standbyTimer = setTimeout(() => setStandby(true), 1e3);
+  }
+}).observe(document.documentElement);
 window.addEventListener("online", connect);
 document.addEventListener("visibilitychange", () => {
   if (document.visibilityState === "hidden") {
@@ -777,6 +814,53 @@ var updateDiagnostics = (stats) => {
     viewerEvents.length ? `viewer events: ${describeEvents(viewerEvents.slice(-5))}` : ""
   ].filter(Boolean).join("\n");
 };
+var reportQuality = (stats) => {
+  if (window.parent === window || !transport || !config || !telemetry) return;
+  const now = performance.now();
+  const current = qualityWindow ??= { at: now, presented: presentedFrames, freezes: inbound?.freezeCount ?? 0, longestGapMs: 0 };
+  const decoded = transport === "webrtc" ? inbound?.framesDecoded ?? 0 : socketDecodedFrames;
+  qualityUnseen = { decoded, presented: presentedFrames, samples: decoded > qualityUnseen.decoded && presentedFrames === qualityUnseen.presented ? qualityUnseen.samples + 1 : 0 };
+  if (qualityUnseen.samples >= 4) current.unseen = true;
+  qualitySamples.push({ stats, telemetry });
+  const windowMs = now - current.at;
+  if (windowMs < qualityWindowMs) return;
+  const samples = qualitySamples;
+  const rungs = samples.map((sample) => sample.telemetry.encoder.rung);
+  qualitySamples = [];
+  qualityWindow = { at: now, presented: presentedFrames, freezes: inbound?.freezeCount ?? 0, longestGapMs: 0, rung: rungs.at(-1) };
+  if (current.unseen) return;
+  const mean = (values) => {
+    const known = values.filter((value) => value !== void 0);
+    return known.length ? Math.round(known.reduce((sum, value) => sum + value, 0) * 10 / known.length) / 10 : void 0;
+  };
+  const displayedFps = Math.round((presentedFrames - current.presented) * 1e4 / windowMs) / 10;
+  const decodedFps = transport === "webrtc" ? mean(samples.map((sample) => sample.stats.videoFps)) : void 0;
+  const rtts = samples.map((sample) => sample.stats.rttMs);
+  window.parent.postMessage({
+    type: DESKTOP_QUALITY_MESSAGE,
+    transport,
+    path: route.path,
+    codec: route.codec,
+    profile: telemetry.encoder.profile,
+    serverVersion: config.serverVersion ?? "unknown",
+    sequence: ++qualitySequence,
+    windowMs: Math.round(windowMs),
+    displayedFps,
+    encoderFps: mean(samples.map((sample) => sample.telemetry.encoder.fps)) ?? 0,
+    longestFrameGapMs: Math.round(Math.max(current.longestGapMs, now - Math.max(presentedAt, current.at))),
+    qualityRung: mean(rungs) ?? 0,
+    worstQualityRung: Math.max(...rungs),
+    qualityDrops: rungs.filter((rung, index) => rung > (index ? rungs[index - 1] : current.rung ?? rung)).length,
+    decodedHeight: screen.videoHeight,
+    inputRttMs: mean(rtts) ?? 0,
+    worstInputRttMs: Math.round(Math.max(...rtts)),
+    cpuLoadPercent: mean(samples.map((sample) => sample.telemetry.cpu.load * 100 / sample.telemetry.cpu.cores)) ?? 0,
+    decodedFps,
+    receiveToDecodeMs: mean(samples.map((sample) => sample.stats.receiverProcessingMs)),
+    lossPercent: mean(samples.map((sample) => sample.stats.lossPercent)),
+    freezes: inbound ? Math.max(0, inbound.freezeCount - current.freezes) : void 0
+  }, "*");
+};
 var inboundOf = (kind) => (stat) => stat.type === "inbound-rtp" && "kind" in stat && stat.kind === kind;
 var reportWebRtcStats = async (pc, stats) => {
   const entries = [...(await pc.getStats()).values()];
@@ -868,6 +952,7 @@ var reportWebRtcStats = async (pc, stats) => {
   if (pair?.availableIncomingBitrate) stats.availableKbps = pair.availableIncomingBitrate / 1e3;
   sendInput({ type: "stats", ...stats }, true);
   updateDiagnostics(stats);
+  reportQuality(stats);
   state.framesDropped = video?.framesDropped;
 };
 setInterval(() => {
@@ -915,6 +1000,7 @@ setInterval(() => {
   else if (transport === "websocket") {
     sendInput({ type: "stats", ...stats });
     updateDiagnostics(stats);
+    reportQuality(stats);
     route.kbps = 0;
   }
 }, 250);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.3.12",
+  "version": "0.3.14",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",