replicas-computer 0.3.11 → 0.3.13

package/assets/desktop/stream-index.html

@@ -15,7 +15,7 @@
     <span id="status-spinner" aria-hidden="true"></span>
     <span id="status-text">connecting…</span>
   </div>
-  <button id="audio-prompt" type="button">Enable audio</button>
+  <button id="audio-prompt" type="button" hidden>Enable audio</button>
   <output id="diagnostics"></output>
   <script type="module" src="./stream-viewer.js?v=__VERSION__"></script>
 </body>

package/dist/index.js

@@ -2937,7 +2937,7 @@ async function computerStatusCommand() {
 }
 
 // src/version.ts
-var COMPUTER_USE_VERSION = "0.3.11";
+var COMPUTER_USE_VERSION = "0.3.13";
 var COMPUTER_USE_PROTOCOL_VERSION = 1;
 
 // src/program.ts

package/dist/stream-viewer.js

@@ -45,6 +45,7 @@ var DESKTOP_WEBRTC_FAILURE_REASONS = {
   video_stalled: "WebRTC video stopped arriving",
   full_chroma_undecodable: "WebRTC could not decode H.264 High 4:4:4"
 };
+var DESKTOP_QUALITY_MESSAGE = "replicas-desktop-quality";
 
 // src/input-validation.ts
 var finite = (value) => typeof value === "number" && Number.isFinite(value);
@@ -100,6 +101,7 @@ diagnostics.classList.toggle("visible", showDiagnostics);
 var webrtcFrameTimeoutMs = 6e3;
 var webrtcSkipsAfterFailure = 3;
 var pointerIntervalMs = 4;
+var qualityWindowMs = 3e4;
 var fullChromaFailureKey = "replicas-desktop-4:4:4-failed";
 var skipFullChroma = false;
 var fullChromaFailure;
@@ -151,6 +153,18 @@ var inboundAt = 0;
 var inboundHistory = [];
 var telemetry;
 var route = { path: "", codec: "", decoded: "", kbps: 0, displayedFps: 0 };
+var qualitySamples = [];
+var qualityWindow;
+var qualityUnseen = { decoded: 0, presented: 0, samples: 0 };
+var qualitySequence = 0;
+var presentedAt = 0;
+var socketDecodedFrames = 0;
+var restartQuality = () => {
+  qualitySamples = [];
+  qualityWindow = void 0;
+  qualitySequence = 0;
+  presentedAt = 0;
+};
 var state = {};
 globalThis.__replicasDesktop = state;
 var viewerEvents = [];
@@ -290,6 +304,7 @@ var configureVideo = async (next, preferHardware = true) => {
   activeVideoConfig = selectedConfig;
   const decoder = new VideoDecoder({
     output: (frame) => {
+      socketDecodedFrames += 1;
       const writer = videoWriter;
       if (!writer || (writer.desiredSize ?? 1) <= 0) {
         frame.close();
@@ -443,6 +458,7 @@ var useFallback = (reasonCode, failed = false) => {
   if (failed) webrtcSkips = webrtcSkipsAfterFailure;
   note("fallback to socket", reason);
   transport = "websocket";
+  restartQuality();
   transportReason = reason;
   route = { ...route, path: "proxy", codec: config.videoCodec };
   announceTransport();
@@ -624,6 +640,7 @@ var connect = () => {
   candidateErrors = 0;
   negotiationErrorName = "";
   lastFrameAt = 0;
+  restartQuality();
   route = { path: "", codec: "", decoded: "", kbps: 0, displayedFps: 0 };
   announceTransport();
   socket?.close();
@@ -685,6 +702,8 @@ var observeFrame = () => {
     presentedFrames += lastPresentedCount ? Math.max(0, metadata.presentedFrames - lastPresentedCount) : 1;
     lastPresentedCount = metadata.presentedFrames;
     lastFrameAt = performance.now();
+    if (presentedAt && qualityWindow) qualityWindow.longestGapMs = Math.max(qualityWindow.longestGapMs, lastFrameAt - Math.max(presentedAt, qualityWindow.at));
+    presentedAt = lastFrameAt;
     status.classList.remove("visible");
     observeFrame();
   });
@@ -777,6 +796,53 @@ var updateDiagnostics = (stats) => {
     viewerEvents.length ? `viewer events: ${describeEvents(viewerEvents.slice(-5))}` : ""
   ].filter(Boolean).join("\n");
 };
+var reportQuality = (stats) => {
+  if (window.parent === window || !transport || !config || !telemetry) return;
+  const now = performance.now();
+  const current = qualityWindow ??= { at: now, presented: presentedFrames, freezes: inbound?.freezeCount ?? 0, longestGapMs: 0 };
+  const decoded = transport === "webrtc" ? inbound?.framesDecoded ?? 0 : socketDecodedFrames;
+  qualityUnseen = { decoded, presented: presentedFrames, samples: decoded > qualityUnseen.decoded && presentedFrames === qualityUnseen.presented ? qualityUnseen.samples + 1 : 0 };
+  if (qualityUnseen.samples >= 4) current.unseen = true;
+  qualitySamples.push({ stats, telemetry });
+  const windowMs = now - current.at;
+  if (windowMs < qualityWindowMs) return;
+  const samples = qualitySamples;
+  const rungs = samples.map((sample) => sample.telemetry.encoder.rung);
+  qualitySamples = [];
+  qualityWindow = { at: now, presented: presentedFrames, freezes: inbound?.freezeCount ?? 0, longestGapMs: 0, rung: rungs.at(-1) };
+  if (current.unseen) return;
+  const mean = (values) => {
+    const known = values.filter((value) => value !== void 0);
+    return known.length ? Math.round(known.reduce((sum, value) => sum + value, 0) * 10 / known.length) / 10 : void 0;
+  };
+  const displayedFps = Math.round((presentedFrames - current.presented) * 1e4 / windowMs) / 10;
+  const decodedFps = transport === "webrtc" ? mean(samples.map((sample) => sample.stats.videoFps)) : void 0;
+  const rtts = samples.map((sample) => sample.stats.rttMs);
+  window.parent.postMessage({
+    type: DESKTOP_QUALITY_MESSAGE,
+    transport,
+    path: route.path,
+    codec: route.codec,
+    profile: telemetry.encoder.profile,
+    serverVersion: config.serverVersion ?? "unknown",
+    sequence: ++qualitySequence,
+    windowMs: Math.round(windowMs),
+    displayedFps,
+    encoderFps: mean(samples.map((sample) => sample.telemetry.encoder.fps)) ?? 0,
+    longestFrameGapMs: Math.round(Math.max(current.longestGapMs, now - Math.max(presentedAt, current.at))),
+    qualityRung: mean(rungs) ?? 0,
+    worstQualityRung: Math.max(...rungs),
+    qualityDrops: rungs.filter((rung, index) => rung > (index ? rungs[index - 1] : current.rung ?? rung)).length,
+    decodedHeight: screen.videoHeight,
+    inputRttMs: mean(rtts) ?? 0,
+    worstInputRttMs: Math.round(Math.max(...rtts)),
+    cpuLoadPercent: mean(samples.map((sample) => sample.telemetry.cpu.load * 100 / sample.telemetry.cpu.cores)) ?? 0,
+    decodedFps,
+    receiveToDecodeMs: mean(samples.map((sample) => sample.stats.receiverProcessingMs)),
+    lossPercent: mean(samples.map((sample) => sample.stats.lossPercent)),
+    freezes: inbound ? Math.max(0, inbound.freezeCount - current.freezes) : void 0
+  }, "*");
+};
 var inboundOf = (kind) => (stat) => stat.type === "inbound-rtp" && "kind" in stat && stat.kind === kind;
 var reportWebRtcStats = async (pc, stats) => {
   const entries = [...(await pc.getStats()).values()];
@@ -868,6 +934,7 @@ var reportWebRtcStats = async (pc, stats) => {
   if (pair?.availableIncomingBitrate) stats.availableKbps = pair.availableIncomingBitrate / 1e3;
   sendInput({ type: "stats", ...stats }, true);
   updateDiagnostics(stats);
+  reportQuality(stats);
   state.framesDropped = video?.framesDropped;
 };
 setInterval(() => {
@@ -915,6 +982,7 @@ setInterval(() => {
   else if (transport === "websocket") {
     sendInput({ type: "stats", ...stats });
     updateDiagnostics(stats);
+    reportQuality(stats);
     route.kbps = 0;
   }
 }, 250);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.3.11",
+  "version": "0.3.13",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",