replicas-computer 0.1.0 → 0.2.1

package/dist/stream-viewer.js

@@ -0,0 +1,308 @@
+// src/media-protocol.ts
+var STREAM_PROTOCOL_VERSION = 1;
+var MEDIA_PACKET_HEADER_BYTES = 12;
+
+// src/stream-viewer.ts
+var screen = document.querySelector("#screen");
+var status = document.querySelector("#status");
+var statusText = document.querySelector("#status-text");
+var audioPrompt = document.querySelector("#audio-prompt");
+var diagnostics = document.querySelector("#diagnostics");
+var TrackGenerator = globalThis.MediaStreamTrackGenerator;
+if (!screen || !status || !statusText || !audioPrompt || !diagnostics) throw new Error("Desktop stream markup is incomplete");
+if (!TrackGenerator) throw new Error("This browser cannot render the desktop stream");
+var params = new URLSearchParams(location.search);
+var showDiagnostics = params.get("diagnostics") === "1";
+diagnostics.classList.toggle("visible", showDiagnostics);
+var socket;
+var config;
+var videoTrack;
+var videoWriter;
+var videoDecoder;
+var activeVideoConfig;
+var videoConfiguration = 0;
+var audioDecoder;
+var audioContext;
+var nextAudioTime = 0;
+var waitingForKeyframe = true;
+var reconnectAttempt = 0;
+var reconnectTimer;
+var pointerFrame;
+var pendingPointer;
+var renderedFrames = 0;
+var decodedAudioPackets = 0;
+var renderedWindowStartedAt = performance.now();
+var rtt = 0;
+var showStatus = (message) => {
+  statusText.textContent = message;
+  status.classList.add("visible");
+};
+var fitScreen = () => {
+  if (!config) return;
+  const scale = Math.min(innerWidth / config.width, innerHeight / config.height);
+  screen.style.width = `${Math.floor(config.width * scale)}px`;
+  screen.style.height = `${Math.floor(config.height * scale)}px`;
+};
+var send = (message) => {
+  if (socket?.readyState === WebSocket.OPEN) socket.send(JSON.stringify(message));
+};
+var resetVideoTrack = () => {
+  void videoWriter?.close().catch(() => {
+  });
+  videoTrack?.stop();
+  videoTrack = new TrackGenerator({ kind: "video" });
+  videoWriter = videoTrack.writable.getWriter();
+  screen.srcObject = new MediaStream([videoTrack]);
+};
+var decoderConfig = (next, preferHardware = false) => {
+  const decoder = {
+    codec: next.videoCodec,
+    codedWidth: next.width,
+    codedHeight: next.height,
+    optimizeForLatency: true,
+    avc: { format: "annexb" }
+  };
+  if (preferHardware) decoder.hardwareAcceleration = "prefer-hardware";
+  return decoder;
+};
+var configureVideo = async (next) => {
+  const configuration = ++videoConfiguration;
+  const previousDecoder = videoDecoder;
+  videoDecoder = void 0;
+  activeVideoConfig = void 0;
+  if (previousDecoder && previousDecoder.state !== "closed") previousDecoder.close();
+  const preferred = decoderConfig(next, true);
+  const support = await VideoDecoder.isConfigSupported(preferred);
+  if (configuration !== videoConfiguration) return;
+  const selectedConfig = support.supported && support.config ? support.config : decoderConfig(next);
+  activeVideoConfig = selectedConfig;
+  const decoder = new VideoDecoder({
+    output: (frame) => {
+      const writer = videoWriter;
+      if (!writer || (writer.desiredSize ?? 1) <= 0) {
+        frame.close();
+        return;
+      }
+      try {
+        void writer.write(frame).catch((error) => showStatus(error instanceof Error ? error.message : String(error))).finally(() => frame.close());
+      } catch (error) {
+        frame.close();
+        showStatus(error instanceof Error ? error.message : String(error));
+      }
+    },
+    error: (error) => {
+      if (videoDecoder !== decoder) return;
+      videoDecoder = void 0;
+      waitingForKeyframe = true;
+      showStatus(`recovering video: ${error.message}`);
+      if (config) void configureVideo(config).catch((reason) => showStatus(reason instanceof Error ? reason.message : String(reason)));
+    }
+  });
+  videoDecoder = decoder;
+  decoder.configure(selectedConfig);
+  waitingForKeyframe = true;
+};
+var enableAudio = async () => {
+  if (!config || typeof AudioDecoder === "undefined") return;
+  audioContext ??= new AudioContext({ latencyHint: "interactive", sampleRate: config.audioSampleRate });
+  await audioContext.resume();
+  if (!audioDecoder || audioDecoder.state === "closed") {
+    const decoderConfig2 = {
+      codec: config.audioCodec,
+      sampleRate: config.audioSampleRate,
+      numberOfChannels: config.audioChannels
+    };
+    const support = await AudioDecoder.isConfigSupported(decoderConfig2);
+    if (!support.supported) throw new Error(`Browser cannot decode ${config.audioCodec}`);
+    audioDecoder = new AudioDecoder({
+      output: (data) => {
+        if (!audioContext || audioContext.state !== "running") {
+          data.close();
+          return;
+        }
+        const buffer = audioContext.createBuffer(data.numberOfChannels, data.numberOfFrames, data.sampleRate);
+        for (let channel = 0; channel < data.numberOfChannels; channel += 1) {
+          data.copyTo(buffer.getChannelData(channel), { planeIndex: channel, format: "f32-planar" });
+        }
+        const source = audioContext.createBufferSource();
+        source.buffer = buffer;
+        source.connect(audioContext.destination);
+        const minimumStart = audioContext.currentTime + 0.035;
+        if (nextAudioTime < minimumStart || nextAudioTime > audioContext.currentTime + 0.25) nextAudioTime = minimumStart;
+        source.start(nextAudioTime);
+        nextAudioTime += data.numberOfFrames / data.sampleRate;
+        data.close();
+        decodedAudioPackets += 1;
+        window.dispatchEvent(new Event("replicasdesktopaudio"));
+      },
+      error: () => {
+        audioPrompt.hidden = false;
+      }
+    });
+    audioDecoder.configure(support.config ?? decoderConfig2);
+  }
+  audioPrompt.hidden = true;
+};
+var handleBinary = (data) => {
+  if (!config || data.byteLength <= MEDIA_PACKET_HEADER_BYTES) return;
+  const view = new DataView(data);
+  if (view.getUint8(1) !== STREAM_PROTOCOL_VERSION) return;
+  const kind = view.getUint8(0);
+  if (kind !== 1 /* VideoKey */ && kind !== 2 /* VideoDelta */ && kind !== 3 /* Audio */) return;
+  const timestamp = Number(view.getBigUint64(4));
+  const payload = new Uint8Array(data, MEDIA_PACKET_HEADER_BYTES);
+  if (kind === 3 /* Audio */) {
+    if (!audioDecoder || audioDecoder.state !== "configured" || audioDecoder.decodeQueueSize > 12) return;
+    audioDecoder.decode(new EncodedAudioChunk({ type: "key", timestamp, data: payload }));
+    return;
+  }
+  if (!videoDecoder || !activeVideoConfig || videoDecoder.state !== "configured") return;
+  const key = kind === 1 /* VideoKey */;
+  if (videoDecoder.decodeQueueSize > 12) {
+    videoDecoder.reset();
+    videoDecoder.configure(activeVideoConfig);
+    waitingForKeyframe = true;
+  }
+  if (waitingForKeyframe && !key) return;
+  waitingForKeyframe = false;
+  try {
+    videoDecoder.decode(new EncodedVideoChunk({ type: key ? "key" : "delta", timestamp, data: payload }));
+  } catch {
+    videoDecoder.reset();
+    videoDecoder.configure(activeVideoConfig);
+    waitingForKeyframe = true;
+  }
+};
+var connect = () => {
+  clearTimeout(reconnectTimer);
+  socket?.close();
+  showStatus(reconnectAttempt ? "reconnecting\u2026" : "connecting\u2026");
+  const nextSocket = new WebSocket(`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/stream`);
+  nextSocket.binaryType = "arraybuffer";
+  socket = nextSocket;
+  nextSocket.addEventListener("open", () => {
+    reconnectAttempt = 0;
+  });
+  nextSocket.addEventListener("message", (event) => {
+    if (event.data instanceof ArrayBuffer) {
+      handleBinary(event.data);
+      return;
+    }
+    if (typeof event.data !== "string") return;
+    const message = JSON.parse(event.data);
+    if (message.type === "pong") {
+      rtt = performance.now() - message.sentAt;
+      return;
+    }
+    if (message.type !== "config" || message.protocolVersion !== STREAM_PROTOCOL_VERSION) return;
+    config = message;
+    fitScreen();
+    resetVideoTrack();
+    void configureVideo(message).catch((error) => showStatus(error instanceof Error ? error.message : String(error)));
+    void screen.play().catch(() => showStatus("Click the stream to begin playback"));
+    enableAudio().catch(() => {
+      audioPrompt.hidden = false;
+    });
+  });
+  nextSocket.addEventListener("close", () => {
+    if (socket !== nextSocket) return;
+    reconnectAttempt += 1;
+    const delay = Math.min(8e3, 250 * 2 ** Math.min(reconnectAttempt, 5));
+    reconnectTimer = setTimeout(connect, delay * (0.85 + Math.random() * 0.3));
+  });
+  nextSocket.addEventListener("error", () => nextSocket.close());
+};
+var normalizedPointer = (event) => {
+  const bounds = screen.getBoundingClientRect();
+  return {
+    x: (event.clientX - bounds.left) / bounds.width,
+    y: (event.clientY - bounds.top) / bounds.height
+  };
+};
+var flushPointer = () => {
+  pointerFrame = void 0;
+  if (!pendingPointer) return;
+  send({ type: "pointer", ...pendingPointer });
+  pendingPointer = void 0;
+};
+var frameCallback = 0;
+var observeFrame = () => {
+  frameCallback = screen.requestVideoFrameCallback(() => {
+    renderedFrames += 1;
+    status.classList.remove("visible");
+    window.dispatchEvent(new Event("replicasdesktopframe"));
+    observeFrame();
+  });
+};
+observeFrame();
+screen.addEventListener("pointermove", (event) => {
+  pendingPointer = normalizedPointer(event);
+  pointerFrame ??= requestAnimationFrame(flushPointer);
+});
+screen.addEventListener("pointerdown", (event) => {
+  event.preventDefault();
+  screen.focus({ preventScroll: true });
+  screen.setPointerCapture(event.pointerId);
+  enableAudio().catch(() => {
+    audioPrompt.hidden = false;
+  });
+  send({ type: "button", ...normalizedPointer(event), button: event.button, down: true });
+});
+screen.addEventListener("pointerup", (event) => {
+  event.preventDefault();
+  send({ type: "button", ...normalizedPointer(event), button: event.button, down: false });
+});
+screen.addEventListener("pointercancel", () => send({ type: "release" }));
+screen.addEventListener("wheel", (event) => {
+  event.preventDefault();
+  send({ type: "wheel", ...normalizedPointer(event), deltaX: event.deltaX, deltaY: event.deltaY });
+}, { passive: false });
+screen.addEventListener("contextmenu", (event) => event.preventDefault());
+window.addEventListener("keydown", (event) => {
+  if (!screen.matches(":focus")) return;
+  event.preventDefault();
+  enableAudio().catch(() => {
+    audioPrompt.hidden = false;
+  });
+  send({ type: "key", code: event.code, down: true, repeat: event.repeat });
+}, { capture: true });
+window.addEventListener("keyup", (event) => {
+  if (!screen.matches(":focus")) return;
+  event.preventDefault();
+  send({ type: "key", code: event.code, down: false });
+}, { capture: true });
+window.addEventListener("blur", () => send({ type: "release" }));
+window.addEventListener("resize", fitScreen);
+window.addEventListener("online", connect);
+window.addEventListener("pagehide", () => {
+  clearTimeout(reconnectTimer);
+  send({ type: "release" });
+  socket?.close();
+  screen.cancelVideoFrameCallback(frameCallback);
+  videoDecoder?.close();
+  audioDecoder?.close();
+  audioContext?.close();
+  void videoWriter?.close();
+  videoTrack?.stop();
+});
+audioPrompt.addEventListener("click", () => enableAudio().catch((error) => showStatus(error instanceof Error ? error.message : String(error))));
+setInterval(() => {
+  if (socket?.readyState === WebSocket.OPEN) send({ type: "ping", sentAt: performance.now() });
+  const now = performance.now();
+  const elapsedSeconds = (now - renderedWindowStartedAt) / 1e3;
+  const measuredFps = renderedFrames / elapsedSeconds;
+  send({
+    type: "stats",
+    videoFps: measuredFps,
+    audioPacketsPerSecond: decodedAudioPackets / elapsedSeconds,
+    decodeQueue: videoDecoder?.decodeQueueSize ?? 0,
+    rttMs: rtt
+  });
+  if (showDiagnostics) {
+    diagnostics.value = `${measuredFps.toFixed(1)} fps \xB7 ${rtt.toFixed(0)} ms RTT \xB7 ${videoDecoder?.decodeQueueSize ?? 0} queued`;
+  }
+  renderedFrames = 0;
+  decodedAudioPackets = 0;
+  renderedWindowStartedAt = now;
+}, 1e3);
+connect();

package/dist/viewer.js

@@ -0,0 +1,199 @@
+// src/viewer.ts
+import RFB from "./core/rfb.js";
+import { encodings } from "./core/encodings.js";
+var screen = document.querySelector("#screen");
+var status = document.querySelector("#status");
+var statusText = document.querySelector("#status-text");
+var hint = document.querySelector("#hint");
+if (!screen || !status || !statusText || !hint) throw new Error("Desktop viewer markup is incomplete");
+var params = new URLSearchParams(location.search);
+var intParam = (name, fallback, min = 0, max = 9) => {
+  const value = Number.parseInt(params.get(name) ?? "", 10);
+  return Number.isInteger(value) && value >= min && value <= max ? value : fallback;
+};
+var requestedProfile = params.get("profile");
+var streamProfile = requestedProfile === "raw" || requestedProfile === "hextile" ? requestedProfile : "tight";
+var requestFps = intParam("requestFps", streamProfile === "tight" ? 120 : 0, 0, 240);
+var requestBatch = intParam("requestBatch", 1, 1, 8);
+var inputRefresh = params.get("inputRefresh") === "1" || streamProfile === "tight";
+var optimisticScroll = params.get("optimisticScroll") === "1";
+var optimisticScrollMax = intParam("optimisticScrollMax", 220, 0, 720);
+var streamDiagnostics = params.get("diagnostics") === "1";
+var modernCursor = 'url("data:image/svg+xml,%3Csvg xmlns=%27http://www.w3.org/2000/svg%27 width=%2724%27 height=%2724%27 viewBox=%270 0 24 24%27%3E%3Cpath d=%27M5 3.5l12.2 11.4-5.8.7 3.2 5.3-2.6 1.5-3.2-5.3-4.1 4.3z%27 fill=%27white%27 stroke=%27%2310161f%27 stroke-width=%272%27 stroke-linejoin=%27round%27/%3E%3C/svg%3E") 5 4, default';
+var wsUrl = `${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/websockify`;
+var activeRfb = null;
+var connectionGeneration = 0;
+var reconnectAttempt = 0;
+var reconnectTimer;
+var updatePump;
+var lastInputRefreshAt = 0;
+var showStatus = (message) => {
+  statusText.textContent = message;
+  status.classList.add("visible");
+};
+var applyModernCursor = () => {
+  screen.style.cursor = modernCursor;
+  const canvas = screen.querySelector("canvas");
+  if (canvas) canvas.style.cursor = modernCursor;
+};
+var requestFramebufferUpdate = (duringContinuousUpdates = false) => {
+  const rfb = activeRfb;
+  if (!rfb || rfb._rfbConnectionState !== "connected" || !duringContinuousUpdates && rfb._enabledContinuousUpdates || !rfb._sock || !rfb._fbWidth || !rfb._fbHeight) return;
+  for (let i = 0; i < requestBatch; i += 1) {
+    RFB.messages.fbUpdateRequest(rfb._sock, true, 0, 0, rfb._fbWidth, rfb._fbHeight);
+  }
+};
+var stopUpdatePump = () => {
+  clearInterval(updatePump);
+  updatePump = void 0;
+};
+var startUpdatePump = () => {
+  stopUpdatePump();
+  if (requestFps) updatePump = setInterval(requestFramebufferUpdate, Math.max(8, Math.round(1e3 / requestFps)));
+};
+var queueInputRefresh = (duringContinuousUpdates = false) => {
+  if (!inputRefresh) return;
+  const now = performance.now();
+  if (now - lastInputRefreshAt < 16) return;
+  lastInputRefreshAt = now;
+  requestFramebufferUpdate(duringContinuousUpdates);
+  for (const delay of [8, 16, 32, 48, 64, 96]) setTimeout(() => requestFramebufferUpdate(duringContinuousUpdates), delay);
+};
+var configureEncodings = (rfb) => {
+  if (streamProfile === "tight") return;
+  const sendDefaultEncodings = rfb._sendEncodings.bind(rfb);
+  rfb._sendEncodings = () => {
+    if (!rfb._sock || rfb._fbDepth !== 24) {
+      sendDefaultEncodings();
+      return;
+    }
+    RFB.messages.clientEncodings(rfb._sock, [
+      encodings.encodingCopyRect,
+      streamProfile === "hextile" ? encodings.encodingHextile : encodings.encodingRaw,
+      encodings.encodingRaw,
+      encodings.pseudoEncodingDesktopSize,
+      encodings.pseudoEncodingLastRect,
+      encodings.pseudoEncodingQEMUExtendedKeyEvent,
+      encodings.pseudoEncodingExtendedDesktopSize,
+      encodings.pseudoEncodingXvp,
+      encodings.pseudoEncodingFence,
+      encodings.pseudoEncodingContinuousUpdates,
+      encodings.pseudoEncodingDesktopName,
+      encodings.pseudoEncodingExtendedClipboard,
+      encodings.pseudoEncodingVMwareCursor,
+      encodings.pseudoEncodingCursor
+    ]);
+  };
+};
+var scheduleReconnect = (immediate = false) => {
+  if (reconnectTimer || document.visibilityState === "hidden" || !navigator.onLine) return;
+  const delay = immediate ? 0 : Math.min(1e4, 250 * 2 ** Math.min(reconnectAttempt, 6)) * (0.8 + Math.random() * 0.4);
+  reconnectAttempt += 1;
+  showStatus(reconnectAttempt > 1 ? "reconnecting\u2026" : "connecting\u2026");
+  reconnectTimer = setTimeout(() => {
+    reconnectTimer = void 0;
+    connect();
+  }, delay);
+};
+var connect = () => {
+  clearTimeout(reconnectTimer);
+  reconnectTimer = void 0;
+  stopUpdatePump();
+  const generation = ++connectionGeneration;
+  const previousRfb = activeRfb;
+  activeRfb = null;
+  previousRfb?.disconnect();
+  showStatus(reconnectAttempt ? "reconnecting\u2026" : "connecting\u2026");
+  let rfb;
+  try {
+    rfb = new RFB(screen, wsUrl);
+  } catch {
+    scheduleReconnect();
+    return;
+  }
+  activeRfb = rfb;
+  rfb.scaleViewport = true;
+  rfb.resizeSession = false;
+  rfb.background = "#000";
+  rfb.showDotCursor = false;
+  if (streamProfile === "tight") {
+    rfb.compressionLevel = intParam("compression", 1);
+    rfb.qualityLevel = intParam("quality", 8);
+  }
+  configureEncodings(rfb);
+  if (streamDiagnostics) {
+    const framebufferUpdate = rfb._framebufferUpdate.bind(rfb);
+    rfb._framebufferUpdate = () => {
+      const complete = framebufferUpdate();
+      if (complete) window.dispatchEvent(new Event("replicasdesktopframe"));
+      return complete;
+    };
+  }
+  rfb.addEventListener("connect", () => {
+    if (generation !== connectionGeneration) return;
+    reconnectAttempt = 0;
+    status.classList.remove("visible");
+    applyModernCursor();
+    startUpdatePump();
+  });
+  rfb.addEventListener("disconnect", () => {
+    if (generation !== connectionGeneration) return;
+    activeRfb = null;
+    stopUpdatePump();
+    scheduleReconnect();
+  });
+  rfb.addEventListener("securityfailure", () => {
+    if (generation !== connectionGeneration) return;
+    activeRfb = null;
+    stopUpdatePump();
+    scheduleReconnect();
+  });
+};
+var shiftCanvasForWheel = (event) => {
+  if (!optimisticScroll || event.ctrlKey || event.metaKey || Math.abs(event.deltaY) <= Math.abs(event.deltaX)) return;
+  const canvas = screen.querySelector("canvas");
+  const context = canvas?.getContext("2d");
+  if (!canvas || !context || !canvas.width || !canvas.height) return;
+  const scale = event.deltaMode === WheelEvent.DOM_DELTA_LINE ? 32 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE ? canvas.height : 1;
+  const limit = Math.max(0, Math.min(optimisticScrollMax, canvas.height - 1));
+  const offset = Math.trunc(Math.max(-limit, Math.min(limit, event.deltaY * scale)));
+  if (offset > 0) {
+    context.drawImage(canvas, 0, offset, canvas.width, canvas.height - offset, 0, 0, canvas.width, canvas.height - offset);
+    context.drawImage(canvas, 0, canvas.height - offset - 1, canvas.width, 1, 0, canvas.height - offset, canvas.width, offset);
+  } else if (offset < 0) {
+    const size = -offset;
+    context.drawImage(canvas, 0, 0, canvas.width, canvas.height - size, 0, size, canvas.width, canvas.height - size);
+    context.drawImage(canvas, 0, size, canvas.width, 1, 0, 0, canvas.width, size);
+  }
+};
+var hintTimer;
+var isMac = /Mac|iPhone|iPad/.test(navigator.userAgentData?.platform ?? navigator.platform ?? "");
+window.addEventListener("keydown", (event) => {
+  if (isMac && event.metaKey && event.key !== "Meta") {
+    hint.classList.add("visible");
+    clearTimeout(hintTimer);
+    hintTimer = setTimeout(() => hint.classList.remove("visible"), 4e3);
+  }
+  queueInputRefresh(true);
+}, { capture: true, passive: true });
+window.addEventListener("wheel", (event) => {
+  shiftCanvasForWheel(event);
+  queueInputRefresh(true);
+}, { capture: true, passive: true });
+for (const eventName of ["pointerdown", "pointerup", "keyup"]) {
+  window.addEventListener(eventName, () => queueInputRefresh(true), { passive: true });
+}
+window.addEventListener("pointermove", () => queueInputRefresh(), { passive: true });
+new MutationObserver(applyModernCursor).observe(screen, { attributes: true, childList: true, subtree: true, attributeFilter: ["style"] });
+window.addEventListener("online", () => scheduleReconnect(true));
+document.addEventListener("visibilitychange", () => {
+  if (document.visibilityState === "visible" && !activeRfb) scheduleReconnect(true);
+});
+window.addEventListener("pagehide", () => {
+  connectionGeneration += 1;
+  clearTimeout(reconnectTimer);
+  stopUpdatePump();
+  activeRfb?.disconnect();
+  activeRfb = null;
+});
+connect();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",
@@ -31,11 +31,15 @@
     "directory": "computer-use"
   },
   "dependencies": {
+    "@novnc/novnc": "1.7.0",
     "chalk": "^4.1.2",
-    "commander": "^12.0.0"
+    "commander": "^12.0.0",
+    "ws": "8.21.2"
   },
   "devDependencies": {
+    "@replicas/shared": "*",
     "@types/node": "^20.11.17",
+    "@types/ws": "8.18.1",
     "tsup": "^8.5.0",
     "typescript": "^5.9.3"
   }