replicas-computer 0.3.3 → 0.3.5

package/README.md

@@ -10,6 +10,7 @@ The versioned computer-use runtime for Replicas workspace images.
 - `src/pacer.ts` owns the per-viewer send budget: fresh media, NACK repairs and protective copies share one token bucket.
 - `src/rtp.ts` owns RTP continuity, keyframe detection, and frame reassembly for the socket fallback.
 - `src/stream-viewer.ts` owns the browser side: WebRTC playback, input routing, the WebSocket fallback, diagnostics and reconnects.
+- `src/input-bridge.ts` converts takeover input into desktop events, accumulating small trackpad movements while preserving full mouse-wheel steps.
 - `src/viewer.ts` owns the optional noVNC fallback.
 - `assets/desktop/start-services.sh` owns the X11 and stream process lifecycle.
 - `xsettingsd` broadcasts theme changes to running GTK apps and Chrome. Chrome follows the dashboard theme without reopening tabs; launch it without a `GTK_THEME` override.
@@ -21,7 +22,7 @@ The viewer and agent command paths are independent. One on-demand encoder pair i
 
 ffmpeg captures the display and encodes 1920×1080 H.264 (and Opus from PulseAudio) as RTP onto loopback. RGB capture is converted with the BT.709 matrix in limited range and the same matrix is signalled in the bitstream, so browsers reconstruct the colours the desktop drew (`REPLICAS_DESKTOP_STREAM_COLOR=legacy` restores the untagged BT.601 conversion). Frames are forwarded unchanged apart from a sequence/timestamp splice and a playout-delay header extension.
 
-WebRTC is the default transport, with automatic WebSocket fallback. A viewer can choose `?transport=websocket`; `REPLICAS_DESKTOP_STREAM_TRANSPORT=websocket` changes the server default for viewers that do not choose. Dashboard URLs carry the selected transport explicitly, and saved transport preferences are preserved. The viewer's WebSocket to `/stream` is the control plane either way: it carries the server's SDP offer, the viewer's answer, trickled ICE candidates, keepalives and telemetry. Pointer moves ride an unreliable data channel. When the server advertises `inputRace`, clicks, keys, wheel events and held-input release race a direct copy against ordered WebSocket delivery. The input bridge accepts only the next action ID, ignoring duplicates and out-of-order direct copies; the socket fills gaps without SCTP retransmission delays or a reorder queue. Older peers keep their existing input path. An explicit server-side WebRTC failure, a network that cannot pass UDP, or six seconds without a displayed frame switches the viewer to the socket, where the server reassembles frames and WebCodecs decodes them in the browser.
+WebRTC is the default transport, with automatic WebSocket fallback. A viewer can choose `?transport=websocket`; `REPLICAS_DESKTOP_STREAM_TRANSPORT=websocket` changes the server default for viewers that do not choose. Dashboard URLs carry the selected transport explicitly, and saved transport preferences are preserved. The viewer's WebSocket to `/stream` is the control plane either way: it carries the server's SDP offer, the viewer's answer, trickled ICE candidates, keepalives and telemetry. Pointer moves ride an unreliable data channel. When the server advertises `inputRace`, clicks, keys, wheel events and held-input release race a direct copy against ordered WebSocket delivery. The input bridge accepts only the next action ID, ignoring duplicates and out-of-order direct copies; the socket fills gaps without SCTP retransmission delays or a reorder queue. Older peers keep their existing input path. The watchdog tracks decoded frames so CSS-hidden viewers do not fall back merely because the browser stops presenting video. An explicit server-side WebRTC failure, a network that cannot pass UDP, or six seconds without decoded video switches the viewer to the socket, where the server reassembles frames and WebCodecs decodes them in the browser.
 
 The server makes the offer, so payload types and SSRCs are fixed and ffmpeg emits them directly. Four H.264 payload types are offered (High 4:4:4 Predictive, High, Main, Constrained Baseline); the encoder runs the best profile every connected viewer accepts, so one viewer without 4:4:4 support moves everyone to 4:2:0, and socket viewers or caps below 8 Mbps do the same. Two profiles select the encoder tuning: `balanced` (CRF 14, default chroma quantisation) and `text` (CRF 10, chroma QP offset −6); `?profile=` picks one and the most recent viewer's choice applies to the shared encoder. Quality adapts on a six-rung bitrate ladder that keeps resolution and frame rate and adds two CRF per rung under the 20 Mbps cap; `?adapt=0` pins the top rung.
 

package/dist/index.js

@@ -2937,7 +2937,7 @@ async function computerStatusCommand() {
 }
 
 // src/version.ts
-var COMPUTER_USE_VERSION = "0.3.3";
+var COMPUTER_USE_VERSION = "0.3.5";
 var COMPUTER_USE_PROTOCOL_VERSION = 1;
 
 // src/program.ts
@@ -3556,6 +3556,7 @@ function superviseEncoder(options) {
 
 // src/input-bridge.ts
 import { spawn as spawn4 } from "child_process";
+var scrollPixelsPerStep = 80;
 var namedKeys = {
   AltLeft: "Alt_L",
   AltRight: "Alt_R",
@@ -3615,6 +3616,8 @@ var InputBridge = class {
   pointerFlush;
   positionSequence = -1;
   actionId = 0;
+  scrollRemainder = 0;
+  scrollButton = 0;
   latestPosition;
   pressedButtons = /* @__PURE__ */ new Set();
   pressedKeys = /* @__PURE__ */ new Set();
@@ -3673,13 +3676,23 @@ var InputBridge = class {
     if (message.type === "wheel") {
       this.pointer = this.coordinates(message.x, message.y);
       this.flushPointer(true);
-      const vertical = Math.min(4, Math.max(1, Math.round(Math.abs(message.deltaY) / 80)));
-      const horizontal = Math.min(4, Math.max(1, Math.round(Math.abs(message.deltaX) / 80)));
-      if (Math.abs(message.deltaY) >= Math.abs(message.deltaX) && message.deltaY) {
-        for (let index = 0; index < vertical; index += 1) this.write(`mouseclick ${message.deltaY < 0 ? 4 : 5}`);
-      } else if (message.deltaX) {
-        for (let index = 0; index < horizontal; index += 1) this.write(`mouseclick ${message.deltaX < 0 ? 6 : 7}`);
+      const vertical = Math.abs(message.deltaY) >= Math.abs(message.deltaX);
+      const delta = vertical ? message.deltaY : message.deltaX;
+      if (!delta) return;
+      const button = vertical ? delta < 0 ? 4 : 5 : delta < 0 ? 6 : 7;
+      if (button !== this.scrollButton) this.scrollRemainder = 0;
+      this.scrollButton = button;
+      const distance = Math.abs(delta);
+      let steps;
+      if (distance >= scrollPixelsPerStep) {
+        steps = Math.min(4, Math.round(distance / scrollPixelsPerStep));
+        this.scrollRemainder = 0;
+      } else {
+        this.scrollRemainder += distance;
+        steps = Math.floor(this.scrollRemainder / scrollPixelsPerStep);
+        this.scrollRemainder %= scrollPixelsPerStep;
       }
+      for (let index = 0; index < steps; index += 1) this.write(`mouseclick ${button}`);
       return;
     }
     if (message.type === "key") {
@@ -3701,6 +3714,8 @@ var InputBridge = class {
     if (this.pointerFlush) clearImmediate(this.pointerFlush);
     this.pointerFlush = void 0;
     this.pointer = void 0;
+    this.scrollRemainder = 0;
+    this.scrollButton = 0;
     for (const button of this.pressedButtons) this.write(`mouseup ${button}`);
     for (const key of this.pressedKeys) this.write(`keyup ${key}`);
     this.pressedButtons.clear();

package/dist/stream-viewer.js

@@ -30,6 +30,12 @@ function validServerMessage(value) {
   }
 }
 
+// src/wheel.ts
+function wheelDeltaPixels(event, pageHeight) {
+  const scale = event.deltaMode === WheelEvent.DOM_DELTA_LINE ? 32 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE ? pageHeight : 1;
+  return { deltaX: event.deltaX * scale, deltaY: event.deltaY * scale };
+}
+
 // ../shared/src/desktop-stream.ts
 var DEFAULT_STREAM_REQUEST = {
   transport: "webrtc",
@@ -506,7 +512,7 @@ var handleServerMessage = (message, viaSocket) => {
     useFallback("recent WebRTC failure");
   } else if (!transport) {
     showStatus("Establishing connection\u2026");
-    webrtcDeadline ??= setTimeout(() => useFallback("WebRTC did not display video in time", true), webrtcFrameTimeoutMs);
+    webrtcDeadline ??= setTimeout(() => useFallback("WebRTC did not decode video in time", true), webrtcFrameTimeoutMs);
   }
 };
 var handleServerText = (data, viaSocket) => {
@@ -578,11 +584,6 @@ var flushPointer = () => {
 var frameCallback = 0;
 var observeFrame = () => {
   frameCallback = screen.requestVideoFrameCallback((_now, metadata) => {
-    const stream = screen.srcObject;
-    if (webrtcDeadline && transport === "webrtc" && stream instanceof MediaStream && peer?.getReceivers().some(({ track }) => stream.getVideoTracks().includes(track))) {
-      clearTimeout(webrtcDeadline);
-      webrtcDeadline = void 0;
-    }
     renderedFrames += 1;
     presentedFrames += lastPresentedCount ? Math.max(0, metadata.presentedFrames - lastPresentedCount) : 1;
     lastPresentedCount = metadata.presentedFrames;
@@ -616,7 +617,7 @@ screen.addEventListener("pointerup", (event) => {
 screen.addEventListener("pointercancel", () => send({ type: "release" }));
 screen.addEventListener("wheel", (event) => {
   event.preventDefault();
-  send({ type: "wheel", ...normalizedPointer(event), deltaX: event.deltaX, deltaY: event.deltaY });
+  send({ type: "wheel", ...normalizedPointer(event), ...wheelDeltaPixels(event, screen.clientHeight) });
 }, { passive: false });
 screen.addEventListener("contextmenu", (event) => event.preventDefault());
 window.addEventListener("keydown", (event) => {
@@ -711,6 +712,11 @@ var reportWebRtcStats = async (pc, stats) => {
     jitterBufferEmittedCount: video?.jitterBufferEmittedCount ?? 0,
     audioPackets: audio?.packetsReceived ?? 0
   };
+  if (counters.framesDecoded > (inbound?.framesDecoded ?? 0)) {
+    lastFrameAt = now;
+    clearTimeout(webrtcDeadline);
+    webrtcDeadline = void 0;
+  }
   if (!inbound) inboundHistory = [];
   while (inboundHistory.length > 1 && inboundHistory[1].at <= now - 1e3) inboundHistory.shift();
   const oldest = inboundHistory[0];
@@ -743,8 +749,6 @@ var reportWebRtcStats = async (pc, stats) => {
   updateDiagnostics(stats);
   state.framesDropped = video?.framesDropped;
 };
-var watchdogPlaybackFrames = 0;
-var watchdogPlaybackAt = 0;
 setInterval(() => {
   if (transport !== "webrtc") return;
   if (screen.paused && screen.srcObject instanceof MediaStream && screen.srcObject.getVideoTracks().some((track) => track.readyState === "live")) return;
@@ -753,14 +757,8 @@ setInterval(() => {
     useFallback("WebRTC video stopped arriving", true);
     return;
   }
-  const quality = screen.getVideoPlaybackQuality();
-  const playbackFrames = quality.totalVideoFrames - quality.droppedVideoFrames;
-  if (playbackFrames !== watchdogPlaybackFrames) {
-    watchdogPlaybackFrames = playbackFrames;
-    watchdogPlaybackAt = now;
-  }
   if (inputChannel?.readyState !== "open") return;
-  if (now - watchdogPlaybackAt < 300 || now - lastFrameAt < 300 || now - lastKeyframeRequestAt < 1e3) return;
+  if (now - lastFrameAt < 2500 || now - lastKeyframeRequestAt < 1e3) return;
   lastKeyframeRequestAt = now;
   note("keyframe requested", `no frame for ${Math.round(now - lastFrameAt)} ms`);
   sendInput({ type: "keyframe" });

package/dist/viewer.js

@@ -1,6 +1,14 @@
 // src/viewer.ts
 import RFB from "./core/rfb.js";
 import { encodings } from "./core/encodings.js";
+
+// src/wheel.ts
+function wheelDeltaPixels(event, pageHeight) {
+  const scale = event.deltaMode === WheelEvent.DOM_DELTA_LINE ? 32 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE ? pageHeight : 1;
+  return { deltaX: event.deltaX * scale, deltaY: event.deltaY * scale };
+}
+
+// src/viewer.ts
 var screen = document.querySelector("#screen");
 var status = document.querySelector("#status");
 var statusText = document.querySelector("#status-text");
@@ -145,9 +153,9 @@ var shiftCanvasForWheel = (event) => {
   const canvas = screen.querySelector("canvas");
   const context = canvas?.getContext("2d");
   if (!canvas || !context || !canvas.width || !canvas.height) return;
-  const scale = event.deltaMode === WheelEvent.DOM_DELTA_LINE ? 32 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE ? canvas.height : 1;
+  const { deltaY } = wheelDeltaPixels(event, canvas.height);
   const limit = Math.max(0, Math.min(optimisticScrollMax, canvas.height - 1));
-  const offset = Math.trunc(Math.max(-limit, Math.min(limit, event.deltaY * scale)));
+  const offset = Math.trunc(Math.max(-limit, Math.min(limit, deltaY)));
   if (offset > 0) {
     context.drawImage(canvas, 0, offset, canvas.width, canvas.height - offset, 0, 0, canvas.width, canvas.height - offset);
     context.drawImage(canvas, 0, canvas.height - offset - 1, canvas.width, 1, 0, canvas.height - offset, canvas.width, offset);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.3.3",
+  "version": "0.3.5",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",