replicas-computer 0.3.3 → 0.3.4

package/README.md

@@ -21,7 +21,7 @@ The viewer and agent command paths are independent. One on-demand encoder pair i
 
 ffmpeg captures the display and encodes 1920×1080 H.264 (and Opus from PulseAudio) as RTP onto loopback. RGB capture is converted with the BT.709 matrix in limited range and the same matrix is signalled in the bitstream, so browsers reconstruct the colours the desktop drew (`REPLICAS_DESKTOP_STREAM_COLOR=legacy` restores the untagged BT.601 conversion). Frames are forwarded unchanged apart from a sequence/timestamp splice and a playout-delay header extension.
 
-WebRTC is the default transport, with automatic WebSocket fallback. A viewer can choose `?transport=websocket`; `REPLICAS_DESKTOP_STREAM_TRANSPORT=websocket` changes the server default for viewers that do not choose. Dashboard URLs carry the selected transport explicitly, and saved transport preferences are preserved. The viewer's WebSocket to `/stream` is the control plane either way: it carries the server's SDP offer, the viewer's answer, trickled ICE candidates, keepalives and telemetry. Pointer moves ride an unreliable data channel. When the server advertises `inputRace`, clicks, keys, wheel events and held-input release race a direct copy against ordered WebSocket delivery. The input bridge accepts only the next action ID, ignoring duplicates and out-of-order direct copies; the socket fills gaps without SCTP retransmission delays or a reorder queue. Older peers keep their existing input path. An explicit server-side WebRTC failure, a network that cannot pass UDP, or six seconds without a displayed frame switches the viewer to the socket, where the server reassembles frames and WebCodecs decodes them in the browser.
+WebRTC is the default transport, with automatic WebSocket fallback. A viewer can choose `?transport=websocket`; `REPLICAS_DESKTOP_STREAM_TRANSPORT=websocket` changes the server default for viewers that do not choose. Dashboard URLs carry the selected transport explicitly, and saved transport preferences are preserved. The viewer's WebSocket to `/stream` is the control plane either way: it carries the server's SDP offer, the viewer's answer, trickled ICE candidates, keepalives and telemetry. Pointer moves ride an unreliable data channel. When the server advertises `inputRace`, clicks, keys, wheel events and held-input release race a direct copy against ordered WebSocket delivery. The input bridge accepts only the next action ID, ignoring duplicates and out-of-order direct copies; the socket fills gaps without SCTP retransmission delays or a reorder queue. Older peers keep their existing input path. The watchdog tracks decoded frames so CSS-hidden viewers do not fall back merely because the browser stops presenting video. An explicit server-side WebRTC failure, a network that cannot pass UDP, or six seconds without decoded video switches the viewer to the socket, where the server reassembles frames and WebCodecs decodes them in the browser.
 
 The server makes the offer, so payload types and SSRCs are fixed and ffmpeg emits them directly. Four H.264 payload types are offered (High 4:4:4 Predictive, High, Main, Constrained Baseline); the encoder runs the best profile every connected viewer accepts, so one viewer without 4:4:4 support moves everyone to 4:2:0, and socket viewers or caps below 8 Mbps do the same. Two profiles select the encoder tuning: `balanced` (CRF 14, default chroma quantisation) and `text` (CRF 10, chroma QP offset −6); `?profile=` picks one and the most recent viewer's choice applies to the shared encoder. Quality adapts on a six-rung bitrate ladder that keeps resolution and frame rate and adds two CRF per rung under the 20 Mbps cap; `?adapt=0` pins the top rung.
 

package/dist/index.js

@@ -2937,7 +2937,7 @@ async function computerStatusCommand() {
 }
 
 // src/version.ts
-var COMPUTER_USE_VERSION = "0.3.3";
+var COMPUTER_USE_VERSION = "0.3.4";
 var COMPUTER_USE_PROTOCOL_VERSION = 1;
 
 // src/program.ts

package/dist/stream-viewer.js

@@ -506,7 +506,7 @@ var handleServerMessage = (message, viaSocket) => {
     useFallback("recent WebRTC failure");
   } else if (!transport) {
     showStatus("Establishing connection\u2026");
-    webrtcDeadline ??= setTimeout(() => useFallback("WebRTC did not display video in time", true), webrtcFrameTimeoutMs);
+    webrtcDeadline ??= setTimeout(() => useFallback("WebRTC did not decode video in time", true), webrtcFrameTimeoutMs);
   }
 };
 var handleServerText = (data, viaSocket) => {
@@ -578,11 +578,6 @@ var flushPointer = () => {
 var frameCallback = 0;
 var observeFrame = () => {
   frameCallback = screen.requestVideoFrameCallback((_now, metadata) => {
-    const stream = screen.srcObject;
-    if (webrtcDeadline && transport === "webrtc" && stream instanceof MediaStream && peer?.getReceivers().some(({ track }) => stream.getVideoTracks().includes(track))) {
-      clearTimeout(webrtcDeadline);
-      webrtcDeadline = void 0;
-    }
     renderedFrames += 1;
     presentedFrames += lastPresentedCount ? Math.max(0, metadata.presentedFrames - lastPresentedCount) : 1;
     lastPresentedCount = metadata.presentedFrames;
@@ -711,6 +706,11 @@ var reportWebRtcStats = async (pc, stats) => {
     jitterBufferEmittedCount: video?.jitterBufferEmittedCount ?? 0,
     audioPackets: audio?.packetsReceived ?? 0
   };
+  if (counters.framesDecoded > (inbound?.framesDecoded ?? 0)) {
+    lastFrameAt = now;
+    clearTimeout(webrtcDeadline);
+    webrtcDeadline = void 0;
+  }
   if (!inbound) inboundHistory = [];
   while (inboundHistory.length > 1 && inboundHistory[1].at <= now - 1e3) inboundHistory.shift();
   const oldest = inboundHistory[0];
@@ -743,8 +743,6 @@ var reportWebRtcStats = async (pc, stats) => {
   updateDiagnostics(stats);
   state.framesDropped = video?.framesDropped;
 };
-var watchdogPlaybackFrames = 0;
-var watchdogPlaybackAt = 0;
 setInterval(() => {
   if (transport !== "webrtc") return;
   if (screen.paused && screen.srcObject instanceof MediaStream && screen.srcObject.getVideoTracks().some((track) => track.readyState === "live")) return;
@@ -753,14 +751,8 @@ setInterval(() => {
     useFallback("WebRTC video stopped arriving", true);
     return;
   }
-  const quality = screen.getVideoPlaybackQuality();
-  const playbackFrames = quality.totalVideoFrames - quality.droppedVideoFrames;
-  if (playbackFrames !== watchdogPlaybackFrames) {
-    watchdogPlaybackFrames = playbackFrames;
-    watchdogPlaybackAt = now;
-  }
   if (inputChannel?.readyState !== "open") return;
-  if (now - watchdogPlaybackAt < 300 || now - lastFrameAt < 300 || now - lastKeyframeRequestAt < 1e3) return;
+  if (now - lastFrameAt < 2500 || now - lastKeyframeRequestAt < 1e3) return;
   lastKeyframeRequestAt = now;
   note("keyframe requested", `no frame for ${Math.round(now - lastFrameAt)} ms`);
   sendInput({ type: "keyframe" });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",