replicas-computer 0.3.5 → 0.3.6

package/dist/index.js

@@ -2937,7 +2937,7 @@ async function computerStatusCommand() {
 }
 
 // src/version.ts
-var COMPUTER_USE_VERSION = "0.3.5";
+var COMPUTER_USE_VERSION = "0.3.6";
 var COMPUTER_USE_PROTOCOL_VERSION = 1;
 
 // src/program.ts
@@ -3755,6 +3755,62 @@ var InputBridge = class {
   }
 };
 
+// ../shared/src/desktop-stream.ts
+var DEFAULT_STREAM_REQUEST = {
+  transport: "webrtc",
+  profile: "balanced",
+  adaptive: true,
+  iceTransportPolicy: "all"
+};
+function parseStreamRequest(query, defaults = DEFAULT_STREAM_REQUEST) {
+  const transport = query.get("transport");
+  const profile = query.get("profile");
+  const adapt = query.get("adapt");
+  const ice = query.get("ice");
+  return {
+    transport: transport === "webrtc" || transport === "websocket" ? transport : defaults.transport,
+    profile: profile === "text" || profile === "balanced" ? profile : defaults.profile,
+    adaptive: adapt === "0" ? false : adapt === "1" ? true : defaults.adaptive,
+    iceTransportPolicy: ice === "relay" || ice === "all" ? ice : defaults.iceTransportPolicy
+  };
+}
+var H264_PROFILES = {
+  high444: { payloadType: 99, profileLevelId: "f4001f", codec: "avc1.F4002A", label: "High 4:4:4" },
+  high: { payloadType: 96, profileLevelId: "64001f", codec: "avc1.64002A", label: "High" },
+  main: { payloadType: 97, profileLevelId: "4d001f", codec: "avc1.4D002A", label: "Main" },
+  baseline: { payloadType: 98, profileLevelId: "42e01f", codec: "avc1.42E02A", label: "Baseline" }
+};
+var PROFILE_ORDER = ["high444", "high", "main", "baseline"];
+var DESKTOP_WEBRTC_FAILURE_REASONS = {
+  server_unavailable: "server has no WebRTC",
+  browser_unavailable: "browser has no WebRTC",
+  connection_failed: "WebRTC connection failed",
+  server_connection_failed: "Server WebRTC connection failed",
+  codec_unsupported: "browser cannot receive H.264 over WebRTC",
+  negotiation_failed: "negotiation failed",
+  video_timeout: "WebRTC did not decode video in time",
+  video_stalled: "WebRTC video stopped arriving"
+};
+function isDesktopWebRtcFailure(value) {
+  if (!isRecord(value) || typeof value.reasonCode !== "string" || !Object.hasOwn(DESKTOP_WEBRTC_FAILURE_REASONS, value.reasonCode) || typeof value.turnConfigured !== "boolean") return false;
+  return [
+    "id",
+    "connectionState",
+    "iceConnectionState",
+    "iceGatheringState",
+    "signalingState",
+    "iceTransportPolicy",
+    "serverVersion",
+    "serverUnavailableReason",
+    "localCandidateTypes",
+    "remoteCandidateTypes",
+    "iceErrorCodes",
+    "errorName",
+    "path",
+    "codec"
+  ].every((key) => typeof value[key] === "string" && value[key].length <= 200) && ["elapsedMs", "candidateErrors", "framesDecoded", "packetsReceived", "packetsLost", "bytesReceived", "lastFrameAgeMs", "encoderFps"].every((key) => typeof value[key] === "number" && Number.isFinite(value[key]));
+}
+
 // src/input-validation.ts
 var finite = (value) => typeof value === "number" && Number.isFinite(value);
 var optionalFinite = (value) => value === void 0 || finite(value);
@@ -3780,7 +3836,7 @@ function validInput(value) {
     case "candidate":
       return typeof value.candidate === "string" && value.candidate.length < 1024 && typeof value.mid === "string" && value.mid.length < 16;
     case "fallback":
-      return typeof value.reason === "string" && value.reason.length < 200;
+      return typeof value.reason === "string" && value.reason.length < 200 && (value.failure === void 0 || isDesktopWebRtcFailure(value.failure));
     case "pointer":
       return finite(value.x) && finite(value.y);
     case "button":
@@ -3801,33 +3857,6 @@ function parseInput(raw) {
   return validInput(value) ? value : void 0;
 }
 
-// ../shared/src/desktop-stream.ts
-var DEFAULT_STREAM_REQUEST = {
-  transport: "webrtc",
-  profile: "balanced",
-  adaptive: true,
-  iceTransportPolicy: "all"
-};
-function parseStreamRequest(query, defaults = DEFAULT_STREAM_REQUEST) {
-  const transport = query.get("transport");
-  const profile = query.get("profile");
-  const adapt = query.get("adapt");
-  const ice = query.get("ice");
-  return {
-    transport: transport === "webrtc" || transport === "websocket" ? transport : defaults.transport,
-    profile: profile === "text" || profile === "balanced" ? profile : defaults.profile,
-    adaptive: adapt === "0" ? false : adapt === "1" ? true : defaults.adaptive,
-    iceTransportPolicy: ice === "relay" || ice === "all" ? ice : defaults.iceTransportPolicy
-  };
-}
-var H264_PROFILES = {
-  high444: { payloadType: 99, profileLevelId: "f4001f", codec: "avc1.F4002A", label: "High 4:4:4" },
-  high: { payloadType: 96, profileLevelId: "64001f", codec: "avc1.64002A", label: "High" },
-  main: { payloadType: 97, profileLevelId: "4d001f", codec: "avc1.4D002A", label: "Main" },
-  baseline: { payloadType: 98, profileLevelId: "42e01f", codec: "avc1.42E02A", label: "Baseline" }
-};
-var PROFILE_ORDER = ["high444", "high", "main", "baseline"];
-
 // src/media-protocol.ts
 var STREAM_PROTOCOL_VERSION = 3;
 var MEDIA_PACKET_HEADER_BYTES = 12;
@@ -4371,6 +4400,7 @@ async function runMediaServer() {
   let videoBytes = 0;
   let audioBytes = 0;
   const events = [];
+  const webrtcFailures = [];
   const samples = [];
   const record = (event, detail) => {
     if (events.push({ at: Date.now(), event, detail }) > eventHistory) events.shift();
@@ -4904,7 +4934,14 @@ async function runMediaServer() {
     record("keyframe", reason);
     void videoEncoder.replace();
   };
-  const enterFallback = (viewer, reason) => {
+  const enterFallback = (viewer, reason, failure) => {
+    if (failure && !viewer.failureReported && viewer.request.transport === "webrtc") {
+      viewer.failureReported = true;
+      const entry = { ...failure, at: Date.now() };
+      if (webrtcFailures.push(entry) > 50) webrtcFailures.shift();
+      process.stderr.write(`[desktop-stream] webrtc fallback diagnostics: ${JSON.stringify(entry)}
+`);
+    }
     if (viewer.fallback) return;
     viewer.fallback = true;
     viewer.waitingForKeyframe = true;
@@ -5001,7 +5038,7 @@ async function runMediaServer() {
         viewer.rtc?.candidate(message.candidate, message.mid);
         return;
       case "fallback":
-        enterFallback(viewer, message.reason);
+        enterFallback(viewer, message.reason, message.failure);
         return;
       case "keyframe":
         if (!viewer.fallback) requestKeyframe("viewer watchdog", void 0, viewer);
@@ -5016,6 +5053,8 @@ async function runMediaServer() {
   const config = {
     type: "config",
     protocolVersion: STREAM_PROTOCOL_VERSION,
+    serverVersion: COMPUTER_USE_VERSION,
+    ...!webrtc ? { webrtcUnavailableReason: webrtcEnabled ? "native_module_unavailable" : "disabled" } : {},
     inputRace: true,
     width,
     height,
@@ -5070,7 +5109,7 @@ async function runMediaServer() {
         width,
         height,
         fps,
-        ...url.pathname === "/telemetry" ? { events, samples } : {}
+        ...url.pathname === "/telemetry" ? { events, samples, webrtcFailures } : {}
       }));
       return;
     }
@@ -5168,9 +5207,10 @@ async function runMediaServer() {
         });
       } catch (error) {
         record("webrtc error", `could not create a peer connection: ${error instanceof Error ? error.message : String(error)}`);
+        reply({ type: "webrtc-failed" });
       }
     }
-    if (!viewer.rtc) enterFallback(viewer, offerWebRtc ? "webrtc unavailable" : "websocket requested");
+    if (!viewer.rtc) enterFallback(viewer, request.transport === "webrtc" ? "webrtc unavailable" : "websocket requested");
     ensureEncoders();
     socket.on("message", (raw, binary) => {
       if (binary) return;

package/dist/stream-viewer.js

@@ -3,39 +3,6 @@ function isRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
-// src/input-validation.ts
-var finite = (value) => typeof value === "number" && Number.isFinite(value);
-var iceServer = (value) => {
-  if (!isRecord(value)) return false;
-  return typeof value.urls === "string" && (value.username === void 0 || typeof value.username === "string") && (value.credential === void 0 || typeof value.credential === "string");
-};
-function validServerMessage(value) {
-  if (!isRecord(value) || typeof value.type !== "string") return false;
-  switch (value.type) {
-    case "ping":
-    case "pong":
-      return finite(value.sentAt);
-    case "webrtc-failed":
-      return true;
-    case "offer":
-      return typeof value.sdp === "string" && value.sdp.length < 65536;
-    case "candidate":
-      return typeof value.candidate === "string" && value.candidate.length < 1024 && typeof value.mid === "string" && value.mid.length < 16;
-    case "config":
-      return finite(value.protocolVersion) && (value.inputRace === void 0 || typeof value.inputRace === "boolean") && finite(value.width) && finite(value.height) && finite(value.fps) && typeof value.videoCodec === "string" && typeof value.audioCodec === "string" && finite(value.audioSampleRate) && finite(value.audioChannels) && isRecord(value.request) && (value.iceServers === void 0 || Array.isArray(value.iceServers) && value.iceServers.every(iceServer));
-    case "telemetry":
-      return isRecord(value.encoder) && isRecord(value.cpu) && isRecord(value.viewer) && Array.isArray(value.events);
-    default:
-      return false;
-  }
-}
-
-// src/wheel.ts
-function wheelDeltaPixels(event, pageHeight) {
-  const scale = event.deltaMode === WheelEvent.DOM_DELTA_LINE ? 32 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE ? pageHeight : 1;
-  return { deltaX: event.deltaX * scale, deltaY: event.deltaY * scale };
-}
-
 // ../shared/src/desktop-stream.ts
 var DEFAULT_STREAM_REQUEST = {
   transport: "webrtc",
@@ -67,6 +34,49 @@ function h264ProfileLabel(sdpFmtpLine) {
   return match?.label ?? id ?? "";
 }
 var DESKTOP_TRANSPORT_MESSAGE = "replicas-desktop-transport";
+var DESKTOP_WEBRTC_FAILURE_REASONS = {
+  server_unavailable: "server has no WebRTC",
+  browser_unavailable: "browser has no WebRTC",
+  connection_failed: "WebRTC connection failed",
+  server_connection_failed: "Server WebRTC connection failed",
+  codec_unsupported: "browser cannot receive H.264 over WebRTC",
+  negotiation_failed: "negotiation failed",
+  video_timeout: "WebRTC did not decode video in time",
+  video_stalled: "WebRTC video stopped arriving"
+};
+
+// src/input-validation.ts
+var finite = (value) => typeof value === "number" && Number.isFinite(value);
+var iceServer = (value) => {
+  if (!isRecord(value)) return false;
+  return typeof value.urls === "string" && (value.username === void 0 || typeof value.username === "string") && (value.credential === void 0 || typeof value.credential === "string");
+};
+function validServerMessage(value) {
+  if (!isRecord(value) || typeof value.type !== "string") return false;
+  switch (value.type) {
+    case "ping":
+    case "pong":
+      return finite(value.sentAt);
+    case "webrtc-failed":
+      return true;
+    case "offer":
+      return typeof value.sdp === "string" && value.sdp.length < 65536;
+    case "candidate":
+      return typeof value.candidate === "string" && value.candidate.length < 1024 && typeof value.mid === "string" && value.mid.length < 16;
+    case "config":
+      return finite(value.protocolVersion) && (value.serverVersion === void 0 || typeof value.serverVersion === "string") && (value.webrtcUnavailableReason === void 0 || typeof value.webrtcUnavailableReason === "string") && (value.inputRace === void 0 || typeof value.inputRace === "boolean") && finite(value.width) && finite(value.height) && finite(value.fps) && typeof value.videoCodec === "string" && typeof value.audioCodec === "string" && finite(value.audioSampleRate) && finite(value.audioChannels) && isRecord(value.request) && (value.iceServers === void 0 || Array.isArray(value.iceServers) && value.iceServers.every(iceServer));
+    case "telemetry":
+      return isRecord(value.encoder) && isRecord(value.cpu) && isRecord(value.viewer) && Array.isArray(value.events);
+    default:
+      return false;
+  }
+}
+
+// src/wheel.ts
+function wheelDeltaPixels(event, pageHeight) {
+  const scale = event.deltaMode === WheelEvent.DOM_DELTA_LINE ? 32 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE ? pageHeight : 1;
+  return { deltaX: event.deltaX * scale, deltaY: event.deltaY * scale };
+}
 
 // src/media-protocol.ts
 var STREAM_PROTOCOL_VERSION = 3;
@@ -138,6 +148,13 @@ globalThis.__replicasDesktop = state;
 var viewerEvents = [];
 state.events = viewerEvents;
 var decoderResets = 0;
+var connectionStartedAt = performance.now();
+var failure;
+var localCandidateTypes = /* @__PURE__ */ new Set();
+var remoteCandidateTypes = /* @__PURE__ */ new Set();
+var iceErrorCodes = /* @__PURE__ */ new Set();
+var candidateErrors = 0;
+var negotiationErrorName = "";
 var note = (event, detail) => {
   if (viewerEvents.push({ at: Date.now(), event, detail }) > 50) viewerEvents.shift();
 };
@@ -208,7 +225,7 @@ var sendInput = (message, unreliable = false) => {
 };
 var announceTransport = () => {
   if (window.parent === window) return;
-  window.parent.postMessage({ type: DESKTOP_TRANSPORT_MESSAGE, transport: transport ?? "connecting", reason: transportReason, path: route.path, codec: route.codec }, "*");
+  window.parent.postMessage({ type: DESKTOP_TRANSPORT_MESSAGE, transport: transport ?? "connecting", reason: transportReason, path: route.path, codec: route.codec, failure }, "*");
 };
 var closePeer = (preserveDeadline = false) => {
   if (!preserveDeadline) {
@@ -379,8 +396,38 @@ var unlockAudio = () => {
     });
   });
 };
-var useFallback = (reason, failed = false) => {
+var useFallback = (reasonCode, failed = false) => {
   if (transport === "websocket" || !config) return;
+  const reason = reasonCode === "websocket_selected" ? "WebSocket transport selected" : reasonCode === "recent_failure" ? "recent WebRTC failure" : DESKTOP_WEBRTC_FAILURE_REASONS[reasonCode];
+  if (request.transport === "webrtc" && reasonCode !== "websocket_selected" && reasonCode !== "recent_failure") {
+    failure = {
+      id: crypto.randomUUID?.() ?? `${Date.now()}-${Math.random().toString(36).slice(2)}`,
+      reasonCode,
+      elapsedMs: Math.round(performance.now() - connectionStartedAt),
+      connectionState: peer?.connectionState ?? "unavailable",
+      iceConnectionState: peer?.iceConnectionState ?? "unavailable",
+      iceGatheringState: peer?.iceGatheringState ?? "unavailable",
+      signalingState: peer?.signalingState ?? "unavailable",
+      iceTransportPolicy: request.iceTransportPolicy,
+      turnConfigured: config.iceServers?.some((server) => /^turns?:/i.test(server.urls)) ?? false,
+      serverVersion: config.serverVersion ?? "unknown",
+      serverUnavailableReason: config.webrtcUnavailableReason ?? "",
+      localCandidateTypes: [...localCandidateTypes].join(","),
+      remoteCandidateTypes: [...remoteCandidateTypes].join(","),
+      iceErrorCodes: [...iceErrorCodes].join(","),
+      candidateErrors,
+      errorName: negotiationErrorName,
+      path: route.path,
+      codec: route.codec,
+      framesDecoded: inbound?.framesDecoded ?? 0,
+      packetsReceived: inbound?.packetsReceived ?? 0,
+      packetsLost: inbound?.packetsLost ?? 0,
+      bytesReceived: inbound?.bytesReceived ?? 0,
+      lastFrameAgeMs: lastFrameAt ? Math.round(performance.now() - lastFrameAt) : -1,
+      encoderFps: telemetry?.encoder.fps ?? -1
+    };
+    state.lastWebRtcFailure = failure;
+  }
   closePeer();
   if (failed) webrtcSkips = webrtcSkipsAfterFailure;
   note("fallback to socket", reason);
@@ -388,11 +435,11 @@ var useFallback = (reason, failed = false) => {
   transportReason = reason;
   route = { ...route, path: "proxy", codec: config.videoCodec };
   announceTransport();
+  sendSocket({ type: "fallback", reason, failure });
   if (!TrackGenerator) {
     showStatus("This browser cannot render the desktop stream");
     return;
   }
-  sendSocket({ type: "fallback", reason });
   resetVideoTrack();
   void configureVideo(config).catch((error) => showStatus(describe(error)));
   void screen.play().catch(() => showStatus("Click the stream to begin playback"));
@@ -427,8 +474,14 @@ var negotiate = async (sdp) => {
   };
   pc.onicecandidate = (event) => {
     if (peer !== pc) return;
+    if (event.candidate?.type) localCandidateTypes.add(event.candidate.type);
     if (event.candidate?.candidate) sendSocket({ type: "candidate", candidate: event.candidate.candidate, mid: event.candidate.sdpMid ?? "0" });
   };
+  pc.onicecandidateerror = (event) => {
+    if (peer !== pc) return;
+    iceErrorCodes.add(event.errorCode);
+    note("ice candidate error", String(event.errorCode));
+  };
   pc.oniceconnectionstatechange = () => {
     if (peer === pc) note("ice", pc.iceConnectionState);
   };
@@ -444,7 +497,7 @@ var negotiate = async (sdp) => {
       unlockAudio();
     } else if (pc.connectionState === "failed") {
       if (transport === "webrtc") socket?.close();
-      else useFallback("WebRTC connection failed", true);
+      else useFallback("connection_failed", true);
     }
   };
   try {
@@ -456,7 +509,7 @@ var negotiate = async (sdp) => {
     if (peer !== pc || !answer.sdp) return;
     const video = answer.sdp.split(/^m=/m).find((section) => section.startsWith("video")) ?? "";
     if (video.startsWith("video 0 ") || !/^a=rtpmap:\d+ H264\/90000/m.test(video)) {
-      useFallback("browser cannot receive H.264 over WebRTC", true);
+      useFallback("codec_unsupported", true);
       return;
     }
     sendSocket({ type: "answer", sdp: answer.sdp });
@@ -484,15 +537,22 @@ var handleServerMessage = (message, viaSocket) => {
     return;
   }
   if (message.type === "webrtc-failed") {
-    useFallback("Server WebRTC connection failed", true);
+    useFallback("server_connection_failed", true);
     return;
   }
   if (message.type === "offer") {
-    void negotiate(message.sdp).catch((error) => useFallback(`negotiation failed: ${describe(error)}`, true));
+    void negotiate(message.sdp).catch((error) => {
+      negotiationErrorName = error instanceof Error ? error.name : "unknown";
+      useFallback("negotiation_failed", true);
+    });
     return;
   }
   if (message.type === "candidate") {
-    void peer?.addIceCandidate({ candidate: message.candidate, sdpMid: message.mid }).catch(() => {
+    const candidateType = message.candidate.match(/\btyp (host|srflx|prflx|relay)\b/)?.[1];
+    if (candidateType) remoteCandidateTypes.add(candidateType);
+    const pc = peer;
+    void pc?.addIceCandidate({ candidate: message.candidate, sdpMid: message.mid }).catch(() => {
+      if (peer === pc) candidateErrors += 1;
     });
     return;
   }
@@ -504,15 +564,15 @@ var handleServerMessage = (message, viaSocket) => {
   config = message;
   request = message.request;
   fitScreen();
-  if (request.transport !== "webrtc") useFallback("WebSocket transport selected");
-  else if (!message.iceServers) useFallback("server has no WebRTC");
-  else if (typeof RTCPeerConnection !== "function") useFallback("browser has no WebRTC");
+  if (request.transport !== "webrtc") useFallback("websocket_selected");
+  else if (!message.iceServers) useFallback("server_unavailable");
+  else if (typeof RTCPeerConnection !== "function") useFallback("browser_unavailable");
   else if (webrtcSkips > 0) {
     webrtcSkips -= 1;
-    useFallback("recent WebRTC failure");
+    useFallback("recent_failure");
   } else if (!transport) {
     showStatus("Establishing connection\u2026");
-    webrtcDeadline ??= setTimeout(() => useFallback("WebRTC did not decode video in time", true), webrtcFrameTimeoutMs);
+    webrtcDeadline ??= setTimeout(() => useFallback("video_timeout", true), webrtcFrameTimeoutMs);
   }
 };
 var handleServerText = (data, viaSocket) => {
@@ -529,6 +589,19 @@ var connect = () => {
   clearTimeout(reconnectTimer);
   closePeer();
   transport = void 0;
+  transportReason = "";
+  failure = void 0;
+  config = void 0;
+  telemetry = void 0;
+  connectionStartedAt = performance.now();
+  localCandidateTypes.clear();
+  remoteCandidateTypes.clear();
+  iceErrorCodes.clear();
+  candidateErrors = 0;
+  negotiationErrorName = "";
+  lastFrameAt = 0;
+  route = { path: "", codec: "", decoded: "", kbps: 0, displayedFps: 0 };
+  announceTransport();
   socket?.close();
   showStatus(reconnectAttempt ? "reconnecting\u2026" : "connecting\u2026");
   const query = new URLSearchParams([...params].filter(([key]) => ["transport", "profile", "adapt", "ice"].includes(key)));
@@ -754,7 +827,7 @@ setInterval(() => {
   if (screen.paused && screen.srcObject instanceof MediaStream && screen.srcObject.getVideoTracks().some((track) => track.readyState === "live")) return;
   const now = performance.now();
   if (now - lastFrameAt >= webrtcFrameTimeoutMs) {
-    useFallback("WebRTC video stopped arriving", true);
+    useFallback("video_stalled", true);
     return;
   }
   if (inputChannel?.readyState !== "open") return;
@@ -778,7 +851,7 @@ setInterval(() => {
     displayedWindow = { at: now, frames: presentedFrames };
   }
   const elapsedSeconds = (now - renderedWindowStartedAt) / 1e3;
-  if (transport !== "webrtc" && elapsedSeconds < 1) return;
+  if (!peer && transport !== "webrtc" && elapsedSeconds < 1) return;
   const stats = {
     sequence: ++statsSequence,
     videoFps: renderedFrames / elapsedSeconds,
@@ -789,7 +862,7 @@ setInterval(() => {
   renderedFrames = 0;
   decodedAudioPackets = 0;
   renderedWindowStartedAt = now;
-  if (transport === "webrtc" && peer) void reportWebRtcStats(peer, stats).catch(() => {
+  if (peer) void reportWebRtcStats(peer, stats).catch(() => {
   });
   else if (transport === "websocket") {
     sendInput({ type: "stats", ...stats });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",