replicas-computer 0.2.3 → 0.2.4

package/dist/index.js

@@ -18,7 +18,7 @@ import { dirname, isAbsolute, resolve } from "path";
 var DESKTOP_STREAM_PORT = 6080;
 function getDesktopViewerUrl(publicUrl) {
   const url = new URL(`${publicUrl.replace(/\/$/, "")}/`);
-  url.searchParams.set("v", "14");
+  url.searchParams.set("v", "15");
   return url.toString();
 }
 var DESKTOP_VIEWER_WIDTH = 1920;
@@ -2921,7 +2921,7 @@ async function computerStatusCommand() {
 }
 
 // src/version.ts
-var COMPUTER_USE_VERSION = "0.2.3";
+var COMPUTER_USE_VERSION = "0.2.4";
 var COMPUTER_USE_PROTOCOL_VERSION = 1;
 
 // src/program.ts
@@ -3357,7 +3357,16 @@ var RtpClock = class {
   base;
   last;
   wraps = 0;
+  origin = 0;
+  emitted = 0;
+  /**
+   * A fresh encoder starts RTP timestamps from a new random base, so the
+   * timeline continues from the last value emitted rather than jumping
+   * backwards — decoders and the viewer's media element both require
+   * monotonic presentation timestamps.
+   */
   reset() {
+    this.origin = this.emitted;
     this.base = void 0;
     this.last = void 0;
     this.wraps = 0;
@@ -3367,7 +3376,8 @@ var RtpClock = class {
     this.last = timestamp;
     const extended = timestamp + this.wraps * 4294967296;
     this.base ??= extended;
-    return Math.round((extended - this.base) * 1e6 / this.rate);
+    this.emitted = this.origin + Math.round((extended - this.base) * 1e6 / this.rate);
+    return this.emitted;
   }
 };
 var startCode = new Uint8Array([0, 0, 0, 1]);
@@ -3462,8 +3472,21 @@ import WebSocket2, { WebSocketServer } from "ws";
 var width = Number.parseInt(process.env.REPLICAS_DESKTOP_WIDTH || String(DESKTOP_VIEWER_WIDTH), 10);
 var height = Number.parseInt(process.env.REPLICAS_DESKTOP_HEIGHT || String(DESKTOP_VIEWER_HEIGHT), 10);
 var fps = Number.parseInt(process.env.REPLICAS_DESKTOP_STREAM_FPS || "60", 10);
-var keyframeInterval = fps;
 var port = Number.parseInt(process.env.REPLICAS_DESKTOP_STREAM_PORT || String(DESKTOP_STREAM_PORT), 10);
+var qualityLadder = [
+  { fps, bitrate: process.env.REPLICAS_DESKTOP_STREAM_VIDEO_BITRATE || "12M", crf: 18 },
+  { fps: Math.max(1, Math.round(fps / 2)), bitrate: "6M", crf: 20 },
+  { fps: Math.max(1, Math.round(fps / 2)), bitrate: "3M", crf: 23 },
+  { fps: Math.max(1, Math.round(fps / 4)), bitrate: "1200k", crf: 26 }
+];
+var keyframeMs = 250;
+var probeIntervalMs = 250;
+var targetBacklogMs = 120;
+var congestedBacklogMs = 250;
+var congestedDecodeQueue = 3;
+var probeTimeoutMs = 2e3;
+var fallbackBufferedBytes = 512 * 1024;
+var maximumBufferedBytes = 4 * 1024 * 1024;
 var display = process.env.REPLICAS_DESKTOP_DISPLAY?.replace(/^([^:])/, ":$1") || ":99";
 var videoCodec = "avc1.42E02A";
 var config = {
@@ -3508,7 +3531,7 @@ function validInput(value) {
   if (!value || typeof value !== "object" || !("type" in value) || typeof value.type !== "string") return false;
   if (value.type === "release") return true;
   if (value.type === "stats") return "videoFps" in value && typeof value.videoFps === "number" && Number.isFinite(value.videoFps) && "audioPacketsPerSecond" in value && typeof value.audioPacketsPerSecond === "number" && Number.isFinite(value.audioPacketsPerSecond) && "decodeQueue" in value && typeof value.decodeQueue === "number" && Number.isFinite(value.decodeQueue) && "rttMs" in value && typeof value.rttMs === "number" && Number.isFinite(value.rttMs);
-  if (value.type === "ping") return "sentAt" in value && typeof value.sentAt === "number";
+  if (value.type === "ping" || value.type === "pong") return "sentAt" in value && typeof value.sentAt === "number";
   if (value.type === "key") return "code" in value && typeof value.code === "string" && value.code.length < 40 && "down" in value && typeof value.down === "boolean";
   if (!("x" in value) || !("y" in value) || typeof value.x !== "number" || typeof value.y !== "number") return false;
   if (value.type === "pointer") return true;
@@ -3524,23 +3547,30 @@ async function runMediaServer() {
   const videoAssembler = new H264RtpAssembler();
   const audioClock = new RtpClock(48e3);
   const clients = /* @__PURE__ */ new Map();
-  let encoder;
-  let encoderRestart;
   let encoderStop;
+  let rung = 0;
+  let descents = 0;
+  let congestedSeconds = 0;
+  let comfortableSeconds = 0;
   let lastVideoAt = 0;
   let lastAudioAt = 0;
   let videoFrames = 0;
   let audioPackets = 0;
   let videoBytes = 0;
   let audioBytes = 0;
+  const queueMs = (state) => {
+    const outstanding = state.probeSentAt ? Date.now() - state.probeSentAt : 0;
+    if (!state.probeAnswered && outstanding > probeTimeoutMs) return 0;
+    return Math.max(state.probeAckedMs, outstanding);
+  };
   const broadcast = (packet, keyframe = false) => {
     for (const [client, state] of clients) {
       if (client.readyState !== WebSocket2.OPEN) continue;
-      if (client.bufferedAmount > 4 * 1024 * 1024) {
+      if (client.bufferedAmount > maximumBufferedBytes) {
         client.close(1013, "stream client is too slow");
         continue;
       }
-      if (!keyframe && (state.waitingForKeyframe || client.bufferedAmount > 512 * 1024)) {
+      if (!keyframe && (state.waitingForKeyframe || queueMs(state) > targetBacklogMs || client.bufferedAmount > fallbackBufferedBytes)) {
         state.waitingForKeyframe = true;
         continue;
       }
@@ -3567,14 +3597,77 @@ async function runMediaServer() {
     audioBytes += rtp.payload.byteLength;
     broadcast(mediaPacket(3 /* Audio */, audioClock.microseconds(rtp.timestamp), rtp.payload));
   });
-  const startEncoder = () => {
-    clearTimeout(encoderStop);
-    if (encoder && encoder.exitCode === null) return;
-    clearTimeout(encoderRestart);
+  const superviseEncoder = (label, prepare, buildArgs) => {
+    let child;
+    let retiring;
+    let restart;
+    let intent = 0;
+    const alive = (process2) => Boolean(process2 && process2.exitCode === null && process2.signalCode === null);
+    const start = () => {
+      clearTimeout(encoderStop);
+      if (alive(child)) return;
+      if (retiring) return;
+      clearTimeout(restart);
+      prepare();
+      const spawned = spawn4("ffmpeg", buildArgs(), { env: { ...process.env, DISPLAY: display }, stdio: ["ignore", "ignore", "pipe"] });
+      spawned.stderr?.on("data", (chunk) => process.stderr.write(`[desktop-stream] ${chunk}`));
+      child = spawned;
+      spawned.once("exit", (code, signal) => {
+        if (child !== spawned) return;
+        child = void 0;
+        if (clients.size === 0) return;
+        process.stderr.write(`[desktop-stream] ${label} encoder exited (${signal || code}); restarting
+`);
+        restart = setTimeout(start, 500);
+      });
+    };
+    return {
+      start,
+      running: () => alive(child),
+      stop: () => {
+        intent += 1;
+        clearTimeout(restart);
+        child?.kill("SIGTERM");
+        retiring?.kill("SIGKILL");
+        child = void 0;
+        retiring = void 0;
+      },
+      replace: () => {
+        const issued = intent += 1;
+        clearTimeout(restart);
+        const previous = child ?? retiring;
+        child = void 0;
+        retiring = void 0;
+        if (!alive(previous)) {
+          start();
+          return;
+        }
+        retiring = previous;
+        let settled = false;
+        const resume = () => {
+          if (settled) return;
+          settled = true;
+          clearTimeout(forceExit);
+          clearTimeout(abandonWait);
+          if (intent !== issued) return;
+          retiring = void 0;
+          if (clients.size === 0) return;
+          start();
+        };
+        const forceExit = setTimeout(() => previous.kill("SIGKILL"), 2e3);
+        const abandonWait = setTimeout(resume, 4e3);
+        previous.once("exit", resume);
+        previous.kill("SIGTERM");
+      }
+    };
+  };
+  const videoEncoder = superviseEncoder("video", () => {
     videoAssembler.reset();
-    audioClock.reset();
-    const videoBitrate = process.env.REPLICAS_DESKTOP_STREAM_VIDEO_BITRATE || "12M";
-    encoder = spawn4("ffmpeg", [
+    for (const state of clients.values()) state.waitingForKeyframe = true;
+  }, () => {
+    const quality = qualityLadder[rung];
+    const keyframeInterval = Math.max(1, Math.round(quality.fps * keyframeMs / 1e3));
+    return [
       "-hide_banner",
       "-loglevel",
       "warning",
@@ -3582,16 +3675,10 @@ async function runMediaServer() {
       "nobuffer",
       "-flags",
       "low_delay",
-      "-thread_queue_size",
-      "1024",
-      "-f",
-      "pulse",
-      "-fragment_size",
-      "2048",
-      "-use_wallclock_as_timestamps",
-      "1",
-      "-i",
-      process.env.REPLICAS_DESKTOP_STREAM_AUDIO_SOURCE || "default",
+      "-probesize",
+      "32",
+      "-analyzeduration",
+      "0",
       "-thread_queue_size",
       "1024",
       "-f",
@@ -3599,15 +3686,13 @@ async function runMediaServer() {
       "-draw_mouse",
       "0",
       "-framerate",
-      String(fps),
+      String(quality.fps),
       "-video_size",
       `${width}x${height}`,
       "-use_wallclock_as_timestamps",
       "1",
       "-i",
       `${display}.0`,
-      "-map",
-      "1:v:0",
       "-an",
       "-c:v",
       "libx264",
@@ -3630,9 +3715,9 @@ async function runMediaServer() {
       "-sc_threshold",
       "0",
       "-crf",
-      "18",
+      String(quality.crf),
       "-maxrate",
-      videoBitrate,
+      quality.bitrate,
       "-bufsize",
       "1M",
       "-x264-params",
@@ -3645,57 +3730,111 @@ async function runMediaServer() {
       "rtp",
       "-payload_type",
       "96",
-      `rtp://127.0.0.1:${videoPort}?pkt_size=1200&connect=1&rtcpport=${videoPort}`,
-      "-map",
-      "0:a:0",
-      "-vn",
-      "-af",
-      "aresample=async=1:first_pts=0",
-      "-c:a",
-      "libopus",
-      "-application",
-      "lowdelay",
-      "-frame_duration",
-      "20",
-      "-b:a",
-      "128k",
-      "-ar",
-      "48000",
-      "-ac",
-      "2",
-      "-flush_packets",
-      "1",
-      "-f",
-      "rtp",
-      "-payload_type",
-      "111",
-      `rtp://127.0.0.1:${audioPort}?pkt_size=1200&connect=1&rtcpport=${audioPort}`
-    ], { env: { ...process.env, DISPLAY: display }, stdio: ["ignore", "ignore", "pipe"] });
-    encoder.stderr?.on("data", (chunk) => process.stderr.write(`[desktop-stream] ${chunk}`));
-    encoder.once("exit", (code, signal) => {
-      encoder = void 0;
-      if (clients.size > 0) {
-        process.stderr.write(`[desktop-stream] encoder exited (${signal || code}); restarting
+      `rtp://127.0.0.1:${videoPort}?pkt_size=1200&connect=1&rtcpport=${videoPort}`
+    ];
+  });
+  const audioEncoder = superviseEncoder("audio", () => audioClock.reset(), () => [
+    "-hide_banner",
+    "-loglevel",
+    "warning",
+    "-fflags",
+    "nobuffer",
+    "-flags",
+    "low_delay",
+    "-probesize",
+    "32",
+    "-analyzeduration",
+    "0",
+    "-thread_queue_size",
+    "1024",
+    "-f",
+    "pulse",
+    "-fragment_size",
+    "2048",
+    "-use_wallclock_as_timestamps",
+    "1",
+    "-i",
+    process.env.REPLICAS_DESKTOP_STREAM_AUDIO_SOURCE || "default",
+    "-vn",
+    "-af",
+    "aresample=async=1:first_pts=0",
+    "-c:a",
+    "libopus",
+    "-application",
+    "lowdelay",
+    "-frame_duration",
+    "20",
+    "-b:a",
+    "128k",
+    "-ar",
+    "48000",
+    "-ac",
+    "2",
+    "-flush_packets",
+    "1",
+    "-f",
+    "rtp",
+    "-payload_type",
+    "111",
+    `rtp://127.0.0.1:${audioPort}?pkt_size=1200&connect=1&rtcpport=${audioPort}`
+  ]);
+  const startEncoders = () => {
+    videoEncoder.start();
+    audioEncoder.start();
+  };
+  const setRung = (next) => {
+    if (next === rung) return;
+    if (next > rung) descents += 1;
+    rung = next;
+    congestedSeconds = 0;
+    comfortableSeconds = 0;
+    process.stderr.write(`[desktop-stream] quality rung ${rung} (${qualityLadder[rung].fps} fps, ${qualityLadder[rung].bitrate})
 `);
-        encoderRestart = setTimeout(startEncoder, 500);
-      }
-    });
+    videoEncoder.replace();
   };
-  const stopEncoderSoon = () => {
+  const stopEncodersSoon = () => {
     clearTimeout(encoderStop);
     encoderStop = setTimeout(() => {
       if (clients.size > 0) return;
-      encoder?.kill("SIGTERM");
-      encoder = void 0;
+      videoEncoder.stop();
+      audioEncoder.stop();
     }, 5e3);
   };
+  const probe = setInterval(() => {
+    for (const [client, state] of clients) {
+      if (client.readyState !== WebSocket2.OPEN || state.probeSentAt) continue;
+      state.probeSentAt = Date.now();
+      client.send(JSON.stringify({ type: "ping", sentAt: state.probeSentAt }));
+    }
+  }, probeIntervalMs);
+  probe.unref();
+  const control = setInterval(() => {
+    const states = [...clients.values()];
+    if (states.length === 0) return;
+    const struggling = states.filter((state) => queueMs(state) > congestedBacklogMs || (state.stats?.decodeQueue ?? 0) > congestedDecodeQueue).length;
+    if (struggling === states.length) {
+      comfortableSeconds = 0;
+      congestedSeconds += 1;
+      if (congestedSeconds >= 2) setRung(Math.min(qualityLadder.length - 1, rung + 1));
+      return;
+    }
+    congestedSeconds = 0;
+    if (struggling > 0) {
+      comfortableSeconds = 0;
+      return;
+    }
+    comfortableSeconds += 1;
+    if (comfortableSeconds >= Math.min(300, 15 * 2 ** descents)) setRung(Math.max(0, rung - 1));
+  }, 1e3);
+  control.unref();
   const httpServer = createServer2((request, response) => {
     const url = new URL(request.url || "/", "http://localhost");
     if (url.pathname === "/health") {
       response.setHeader("content-type", "application/json");
       response.end(JSON.stringify({
         clients: clients.size,
-        encoder: Boolean(encoder && encoder.exitCode === null),
+        encoder: videoEncoder.running(),
+        audioEncoder: audioEncoder.running(),
         lastVideoFrameMsAgo: lastVideoAt ? Date.now() - lastVideoAt : null,
         lastAudioPacketMsAgo: lastAudioAt ? Date.now() - lastAudioAt : null,
         videoFrames,
@@ -3703,9 +3842,11 @@ async function runMediaServer() {
         videoBytes,
         audioBytes,
         clientStats: [...clients.values()].flatMap((client) => client.stats ? [client.stats] : []),
+        clientQueueMs: [...clients.values()].map((client) => Math.round(queueMs(client))),
+        qualityRung: rung,
         width,
         height,
-        fps
+        fps: qualityLadder[rung].fps
       }));
       return;
     }
@@ -3729,9 +3870,15 @@ async function runMediaServer() {
     webSocketServer.handleUpgrade(request, socket, head, (client) => webSocketServer.emit("connection", client, request));
   });
   webSocketServer.on("connection", (client) => {
-    clients.set(client, { waitingForKeyframe: true, input: new InputBridge(display, width, height) });
+    clients.set(client, {
+      waitingForKeyframe: true,
+      input: new InputBridge(display, width, height),
+      probeSentAt: 0,
+      probeAckedMs: 0,
+      probeAnswered: false
+    });
     client.send(JSON.stringify(config));
-    startEncoder();
+    startEncoders();
     client.on("message", (raw, binary) => {
       if (binary) return;
       let message;
@@ -3742,21 +3889,32 @@ async function runMediaServer() {
       }
       if (!validInput(message)) return;
       if (message.type === "ping") client.send(JSON.stringify({ type: "pong", sentAt: message.sentAt }));
-      else if (message.type === "stats") {
+      else if (message.type === "pong") {
+        const state = clients.get(client);
+        if (!state || message.sentAt !== state.probeSentAt) return;
+        state.probeAckedMs = Date.now() - state.probeSentAt;
+        state.probeSentAt = 0;
+        state.probeAnswered = true;
+      } else if (message.type === "stats") {
         const state = clients.get(client);
         if (!state) return;
         state.stats = {
-          videoFps: message.videoFps,
-          audioPacketsPerSecond: message.audioPacketsPerSecond,
-          decodeQueue: message.decodeQueue,
-          rttMs: message.rttMs
+          videoFps: clamp(message.videoFps, 0, 1e3),
+          audioPacketsPerSecond: clamp(message.audioPacketsPerSecond, 0, 1e3),
+          decodeQueue: clamp(message.decodeQueue, 0, 1e3),
+          rttMs: clamp(message.rttMs, 0, 6e5)
         };
       } else clients.get(client)?.input.handle(message);
     });
     client.once("close", () => {
       clients.get(client)?.input.close();
       clients.delete(client);
-      if (clients.size === 0) stopEncoderSoon();
+      if (clients.size > 0) return;
+      rung = 0;
+      descents = 0;
+      congestedSeconds = 0;
+      comfortableSeconds = 0;
+      stopEncodersSoon();
     });
   });
   await new Promise((resolveServer, reject) => {
@@ -3772,13 +3930,15 @@ async function runMediaServer() {
     const close = () => {
       if (closing) return;
       closing = true;
-      clearTimeout(encoderRestart);
+      clearInterval(probe);
+      clearInterval(control);
       clearTimeout(encoderStop);
       for (const [client, state] of clients) {
         state.input.close();
         client.close(1001, "server shutting down");
       }
-      encoder?.kill("SIGTERM");
+      videoEncoder.stop();
+      audioEncoder.stop();
       videoSocket.close();
       audioSocket.close();
       webSocketServer.close();

package/dist/stream-viewer.js

@@ -166,7 +166,7 @@ var handleBinary = (data) => {
   }
   if (!videoDecoder || !activeVideoConfig || videoDecoder.state !== "configured") return;
   const key = kind === 1 /* VideoKey */;
-  if (videoDecoder.decodeQueueSize > 12) {
+  if (videoDecoder.decodeQueueSize > 3) {
     videoDecoder.reset();
     videoDecoder.configure(activeVideoConfig);
     waitingForKeyframe = true;
@@ -205,6 +205,10 @@ var connect = () => {
       rtt = performance.now() - message.sentAt;
       return;
     }
+    if (message.type === "ping") {
+      send({ type: "pong", sentAt: message.sentAt });
+      return;
+    }
     if (message.type !== "config" || message.protocolVersion !== STREAM_PROTOCOL_VERSION) return;
     config = message;
     fitScreen();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",