replicas-computer 0.3.13 → 0.3.15

package/dist/stream-viewer.js

@@ -99,6 +99,10 @@ var view = params.get("view") === "native" ? "native" : params.get("view") === "
 document.documentElement.dataset.view = view;
 diagnostics.classList.toggle("visible", showDiagnostics);
 var webrtcFrameTimeoutMs = 6e3;
+var webrtcFirstFrameMs = 3e3;
+var webrtcRetryFirstMs = 15e3;
+var webrtcRetryMaxMs = 3e5;
+var webrtcRetryTimeoutMs = 1e4;
 var webrtcSkipsAfterFailure = 3;
 var pointerIntervalMs = 4;
 var qualityWindowMs = 3e4;
@@ -110,6 +114,8 @@ try {
 } catch {
 }
 var socket;
+var standby = innerWidth === 0 || innerHeight === 0;
+var standbyTimer;
 var config;
 var transport;
 var transportReason = "";
@@ -117,6 +123,7 @@ var peer;
 var inputChannel;
 var pointerChannel;
 var webrtcDeadline;
+var webrtcDeadlineAt = 0;
 var webrtcSkips = 0;
 var videoTrack;
 var videoWriter;
@@ -172,6 +179,7 @@ state.events = viewerEvents;
 var decoderResets = 0;
 var connectionStartedAt = performance.now();
 var failure;
+var recovery;
 var localCandidateTypes = /* @__PURE__ */ new Set();
 var remoteCandidateTypes = /* @__PURE__ */ new Set();
 var iceErrorCodes = /* @__PURE__ */ new Set();
@@ -248,9 +256,10 @@ var sendInput = (message, unreliable = false) => {
 };
 var announceTransport = () => {
   if (window.parent === window) return;
-  window.parent.postMessage({ type: DESKTOP_TRANSPORT_MESSAGE, transport: transport ?? "connecting", reason: transportReason, path: route.path, codec: route.codec, failure }, "*");
+  window.parent.postMessage({ type: DESKTOP_TRANSPORT_MESSAGE, transport: transport ?? "connecting", reason: transportReason, path: route.path, codec: route.codec, failure, recovery }, "*");
 };
 var closePeer = (preserveDeadline = false) => {
+  cancelRetry();
   if (!preserveDeadline) {
     clearTimeout(webrtcDeadline);
     webrtcDeadline = void 0;
@@ -392,7 +401,14 @@ var handleBinary = (data) => {
     waitingForKeyframe = true;
     state.decoderResets = ++decoderResets;
   }
-  if (waitingForKeyframe && !key) return;
+  if (waitingForKeyframe && !key) {
+    const now = performance.now();
+    if (now - lastKeyframeRequestAt >= 1e3) {
+      lastKeyframeRequestAt = now;
+      sendSocket({ type: "keyframe" });
+    }
+    return;
+  }
   waitingForKeyframe = false;
   try {
     videoDecoder.decode(new EncodedVideoChunk({ type: key ? "key" : "delta", timestamp, data: payload }));
@@ -463,6 +479,9 @@ var useFallback = (reasonCode, failed = false) => {
   route = { ...route, path: "proxy", codec: config.videoCodec };
   announceTransport();
   sendSocket({ type: "fallback", reason, failure });
+  socketSince = performance.now();
+  retryAttempts = 0;
+  if (reasonCode !== "websocket_selected" && reasonCode !== "server_unavailable" && reasonCode !== "browser_unavailable" && reasonCode !== "codec_unsupported") scheduleRetry();
   if (!TrackGenerator) {
     showStatus("This browser cannot render the desktop stream");
     return;
@@ -475,35 +494,15 @@ var useFallback = (reasonCode, failed = false) => {
 var negotiate = async (sdp) => {
   if (!config?.iceServers || transport === "websocket") return;
   closePeer(true);
-  const pc = new RTCPeerConnection({ iceServers: config.iceServers, bundlePolicy: "max-bundle", iceTransportPolicy: config.request.iceTransportPolicy });
-  peer = pc;
-  pc.ontrack = (event) => {
-    if (peer !== pc) return;
-    try {
-      event.receiver.jitterBufferTarget = 0;
-    } catch {
-    }
-    try {
-      event.receiver.playoutDelayHint = 0;
-    } catch {
-    }
-    if (screen.srcObject !== event.streams[0]) screen.srcObject = event.streams[0] ?? new MediaStream([event.track]);
+  const pc = openPeer(config, () => peer === pc, (stream) => {
+    if (screen.srcObject !== stream) screen.srcObject = stream;
     void screen.play().catch(() => {
     });
-  };
-  pc.ondatachannel = (event) => {
-    if (peer !== pc) return;
-    if (event.channel.label === "pointer") pointerChannel = event.channel;
-    else inputChannel = event.channel;
-    event.channel.onmessage = (message) => {
-      if (typeof message.data === "string") handleServerText(message.data, false);
-    };
-  };
-  pc.onicecandidate = (event) => {
-    if (peer !== pc) return;
-    if (event.candidate?.type) localCandidateTypes.add(event.candidate.type);
-    if (event.candidate?.candidate) sendSocket({ type: "candidate", candidate: event.candidate.candidate, mid: event.candidate.sdpMid ?? "0" });
-  };
+  }, (channel) => {
+    if (channel.label === "pointer") pointerChannel = channel;
+    else inputChannel = channel;
+  }, (type) => localCandidateTypes.add(type));
+  peer = pc;
   pc.onicecandidateerror = (event) => {
     if (peer !== pc) return;
     iceErrorCodes.add(event.errorCode);
@@ -520,6 +519,7 @@ var negotiate = async (sdp) => {
       transportReason = "";
       webrtcSkips = 0;
       lastFrameAt = performance.now();
+      if (webrtcDeadline && webrtcDeadlineAt - lastFrameAt < webrtcFirstFrameMs) armWebrtcDeadline(webrtcFirstFrameMs);
       announceTransport();
       unlockAudio();
     } else if (pc.connectionState === "failed") {
@@ -528,26 +528,166 @@ var negotiate = async (sdp) => {
     }
   };
   try {
-    await pc.setRemoteDescription({ type: "offer", sdp });
-    if (peer !== pc) return;
-    if (skipFullChroma) {
-      const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs.filter((codec) => h264ProfileLabel(codec.sdpFmtpLine ?? "") !== H264_PROFILES.high444.label);
-      if (codecs) pc.getTransceivers().find((transceiver) => transceiver.receiver.track.kind === "video")?.setCodecPreferences(codecs);
-    }
-    const answer = await pc.createAnswer();
-    if (peer !== pc) return;
-    await pc.setLocalDescription(answer);
-    if (peer !== pc || !answer.sdp) return;
-    const video = answer.sdp.split(/^m=/m).find((section) => section.startsWith("video")) ?? "";
-    if (video.startsWith("video 0 ") || !/^a=rtpmap:\d+ H264\/90000/m.test(video)) {
-      useFallback("codec_unsupported", true);
-      return;
-    }
-    sendSocket({ type: "answer", sdp: answer.sdp });
+    if (!await answerOffer(pc, sdp, skipFullChroma, () => peer === pc)) useFallback("codec_unsupported", true);
   } catch (error) {
     if (peer === pc) throw error;
   }
 };
+var renderOnArrival = (receiver) => {
+  try {
+    receiver.jitterBufferTarget = 0;
+  } catch {
+  }
+  try {
+    receiver.playoutDelayHint = 0;
+  } catch {
+  }
+};
+var openPeer = ({ iceServers, request: { iceTransportPolicy } }, current, play, attach, candidateType) => {
+  const pc = new RTCPeerConnection({ iceServers, bundlePolicy: "max-bundle", iceTransportPolicy });
+  pc.ontrack = (event) => {
+    if (!current()) return;
+    renderOnArrival(event.receiver);
+    play(event.streams[0] ?? new MediaStream([event.track]));
+  };
+  pc.ondatachannel = (event) => {
+    if (!current()) return;
+    attach(event.channel);
+    event.channel.onmessage = (message) => {
+      if (typeof message.data === "string") handleServerText(message.data, false);
+    };
+  };
+  pc.onicecandidate = (event) => {
+    if (!current()) return;
+    if (event.candidate?.type) candidateType?.(event.candidate.type);
+    if (event.candidate?.candidate) sendSocket({ type: "candidate", candidate: event.candidate.candidate, mid: event.candidate.sdpMid ?? "0" });
+  };
+  return pc;
+};
+var answerOffer = async (pc, sdp, withoutFullChroma, current) => {
+  await pc.setRemoteDescription({ type: "offer", sdp });
+  if (!current()) return true;
+  if (withoutFullChroma) {
+    const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs.filter((codec) => h264ProfileLabel(codec.sdpFmtpLine ?? "") !== H264_PROFILES.high444.label);
+    if (codecs) pc.getTransceivers().find((transceiver) => transceiver.receiver.track.kind === "video")?.setCodecPreferences(codecs);
+  }
+  const answer = await pc.createAnswer();
+  if (!current()) return true;
+  await pc.setLocalDescription(answer);
+  if (!current() || !answer.sdp) return true;
+  const video = answer.sdp.split(/^m=/m).find((section) => section.startsWith("video")) ?? "";
+  if (video.startsWith("video 0 ") || !/^a=rtpmap:\d+ H264\/90000/m.test(video)) return false;
+  sendSocket({ type: "answer", sdp: answer.sdp });
+  return true;
+};
+var retry;
+var retryRequested = false;
+var retryTimer;
+var retryDeadline;
+var retryDelayMs = webrtcRetryFirstMs;
+var retryAttempts = 0;
+var socketSince = 0;
+var retryScreen = document.createElement("video");
+retryScreen.muted = true;
+retryScreen.playsInline = true;
+retryScreen.setAttribute("aria-hidden", "true");
+retryScreen.style.cssText = "position:fixed;left:0;top:0;width:2px;height:2px;opacity:0;pointer-events:none";
+document.body.append(retryScreen);
+var cancelRetry = () => {
+  clearTimeout(retryTimer);
+  clearTimeout(retryDeadline);
+  retryTimer = void 0;
+  retryRequested = false;
+  retry?.pc.close();
+  retry = void 0;
+  retryScreen.srcObject = null;
+};
+var scheduleRetry = () => {
+  if (retryTimer || retryRequested || request.transport !== "webrtc" || !config?.iceServers || typeof RTCPeerConnection !== "function") return;
+  retryTimer = setTimeout(() => {
+    retryTimer = void 0;
+    if (transport !== "websocket" || socket?.readyState !== WebSocket.OPEN) return;
+    retryRequested = true;
+    retryAttempts += 1;
+    note("retrying WebRTC", `attempt ${retryAttempts}`);
+    sendSocket({ type: "retry-webrtc" });
+    retryDeadline = setTimeout(() => failRetry("no video in time"), webrtcRetryTimeoutMs);
+  }, retryDelayMs);
+  retryDelayMs = Math.min(webrtcRetryMaxMs, retryDelayMs * 2);
+};
+var failRetry = (reason) => {
+  if (!retryRequested) return;
+  cancelRetry();
+  note("WebRTC retry failed", reason);
+  sendSocket({ type: "fallback", reason: `WebRTC retry: ${reason}` });
+  scheduleRetry();
+};
+var answerRetry = async (sdp) => {
+  if (!retryRequested || retry || !config?.iceServers) return;
+  const pc = openPeer(config, () => retry === attempt, (stream) => {
+    attempt.stream = stream;
+    retryScreen.srcObject = stream;
+    void retryScreen.play().catch(() => {
+    });
+  }, (channel) => {
+    if (channel.label === "pointer") attempt.pointer = channel;
+    else attempt.input = channel;
+  });
+  const attempt = { pc };
+  retry = attempt;
+  pc.onconnectionstatechange = () => {
+    if (peer === pc) {
+      note("webrtc", pc.connectionState);
+      if (pc.connectionState === "failed") socket?.close();
+    } else if (retry === attempt && pc.connectionState === "failed") failRetry("connection failed");
+  };
+  try {
+    if (!await answerOffer(pc, sdp, true, () => retry === attempt)) failRetry("codec unsupported");
+  } catch (error) {
+    if (retry === attempt) failRetry(describe(error));
+  }
+};
+var checkRetry = async (attempt) => {
+  const video = [...(await attempt.pc.getStats()).values()].find(inboundOf("video"));
+  if (retry === attempt && (video?.framesDecoded ?? 0) > 0) completeRetry(attempt);
+};
+var completeRetry = (attempt) => {
+  clearTimeout(retryDeadline);
+  retry = void 0;
+  retryRequested = false;
+  videoConfiguration += 1;
+  if (videoDecoder?.state !== "closed") videoDecoder?.close();
+  videoDecoder = void 0;
+  activeVideoConfig = void 0;
+  if (audioDecoder?.state !== "closed") audioDecoder?.close();
+  audioDecoder = void 0;
+  void videoWriter?.close().catch(() => {
+  });
+  videoTrack?.stop();
+  videoTrack = void 0;
+  videoWriter = void 0;
+  peer = attempt.pc;
+  inputChannel = attempt.input;
+  pointerChannel = attempt.pointer;
+  inbound = void 0;
+  inboundHistory = [];
+  screen.srcObject = attempt.stream ?? null;
+  retryScreen.srcObject = null;
+  recovery = { id: crypto.randomUUID?.() ?? `${Date.now()}`, afterMs: Math.round(performance.now() - socketSince), attempts: retryAttempts };
+  note("back on WebRTC", `${Math.round(recovery.afterMs / 1e3)} s and ${retryAttempts} attempts on the socket`);
+  transport = "webrtc";
+  transportReason = "";
+  failure = void 0;
+  webrtcSkips = 0;
+  retryDelayMs = webrtcRetryFirstMs;
+  retryAttempts = 0;
+  lastFrameAt = performance.now();
+  restartQuality();
+  route = { ...route, path: "", codec: "" };
+  sendSocket({ type: "webrtc-ready" });
+  announceTransport();
+  unlockAudio();
+};
 var handleServerMessage = (message, viaSocket) => {
   if (message.type === "pong") {
     const elapsed = performance.now() - message.sentAt;
@@ -568,10 +708,15 @@ var handleServerMessage = (message, viaSocket) => {
     return;
   }
   if (message.type === "webrtc-failed") {
-    useFallback("server_connection_failed", true);
+    if (retryRequested) failRetry("server connection failed");
+    else useFallback("server_connection_failed", true);
     return;
   }
   if (message.type === "offer") {
+    if (retryRequested) {
+      void answerRetry(message.sdp);
+      return;
+    }
     void negotiate(message.sdp).catch((error) => {
       negotiationErrorName = error instanceof Error ? error.name : "unknown";
       useFallback("negotiation_failed", true);
@@ -581,7 +726,7 @@ var handleServerMessage = (message, viaSocket) => {
   if (message.type === "candidate") {
     const candidateType = message.candidate.match(/\btyp (host|srflx|prflx|relay)\b/)?.[1];
     if (candidateType) remoteCandidateTypes.add(candidateType);
-    const pc = peer;
+    const pc = retry?.pc ?? peer;
     void pc?.addIceCandidate({ candidate: message.candidate, sdpMid: message.mid }).catch(() => {
       if (peer === pc) candidateErrors += 1;
     });
@@ -603,18 +748,24 @@ var handleServerMessage = (message, viaSocket) => {
     useFallback("recent_failure");
   } else if (!transport) {
     showStatus("Establishing connection\u2026", true);
-    webrtcDeadline ??= setTimeout(() => {
-      if (skipFullChroma || !inbound?.packetsReceived || !route.codec.endsWith(H264_PROFILES.high444.label)) {
-        useFallback("video_timeout", true);
-        return;
-      }
-      skipFullChroma = true;
-      if (config) fullChromaFailure = captureFailure("full_chroma_undecodable", config);
-      note("retrying WebRTC without 4:4:4");
-      socket?.close();
-    }, webrtcFrameTimeoutMs);
+    armWebrtcDeadline(webrtcFrameTimeoutMs);
   }
 };
+var armWebrtcDeadline = (ms) => {
+  clearTimeout(webrtcDeadline);
+  webrtcDeadlineAt = performance.now() + ms;
+  webrtcDeadline = setTimeout(() => {
+    webrtcDeadline = void 0;
+    if (skipFullChroma || !inbound?.packetsReceived || !route.codec.endsWith(H264_PROFILES.high444.label)) {
+      useFallback("video_timeout", true);
+      return;
+    }
+    skipFullChroma = true;
+    if (config) fullChromaFailure = captureFailure("full_chroma_undecodable", config);
+    note("retrying WebRTC without 4:4:4");
+    connect();
+  }, ms);
+};
 var handleServerText = (data, viaSocket) => {
   let parsed;
   try {
@@ -631,6 +782,7 @@ var connect = () => {
   transport = void 0;
   transportReason = "";
   failure = void 0;
+  recovery = void 0;
   config = void 0;
   telemetry = void 0;
   connectionStartedAt = performance.now();
@@ -646,6 +798,7 @@ var connect = () => {
   socket?.close();
   showStatus(reconnectAttempt ? "reconnecting\u2026" : "connecting\u2026", true);
   const query = new URLSearchParams([...params].filter(([key]) => ["transport", "profile", "adapt", "ice"].includes(key)));
+  if (standby) query.set("standby", "1");
   const nextSocket = new WebSocket(`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/stream?${query}`);
   nextSocket.binaryType = "arraybuffer";
   socket = nextSocket;
@@ -749,6 +902,21 @@ window.addEventListener("keyup", (event) => {
 }, { capture: true });
 window.addEventListener("blur", () => send({ type: "release" }));
 window.addEventListener("resize", fitScreen);
+var setStandby = (next) => {
+  clearTimeout(standbyTimer);
+  if (next === standby) return;
+  standby = next;
+  note(standby ? "standby" : "active");
+  sendSocket({ type: "standby", standby });
+  if (!standby && !audioPrompt.hidden) unlockAudio();
+};
+new IntersectionObserver(([entry]) => {
+  if (entry.isIntersecting) setStandby(false);
+  else if (!standby) {
+    clearTimeout(standbyTimer);
+    standbyTimer = setTimeout(() => setStandby(true), 1e3);
+  }
+}).observe(document.documentElement);
 window.addEventListener("online", connect);
 document.addEventListener("visibilitychange", () => {
   if (document.visibilityState === "hidden") {
@@ -789,7 +957,7 @@ var updateDiagnostics = (stats) => {
     `source ${dims.source} \xB7 decoded ${dims.decoded || "\u2014"} \xB7 css ${dims.css} \xB7 device ${dims.physical}`,
     `${route.displayedFps.toFixed(1)} displayed/s \xB7 ${stats.videoFps.toFixed(1)} decoded/s \xB7 ${route.kbps.toFixed(0)} kbps \xB7 ${stats.rttMs.toFixed(0)} ms rtt \xB7 ${socketRtt.toFixed(0)} ms via proxy \xB7 ${network}`,
     transport === "webrtc" ? `receiver: ${stats.decodeMs?.toFixed(2) ?? "\u2014"} ms decode \xB7 ${stats.receiverProcessingMs?.toFixed(2) ?? "\u2014"} ms receive \u2192 decoded (includes buffer + decode; excludes display)` : "",
-    encoder ? `encoder ${encoder.profile} crf ${encoder.crf} \xB7 ${encoder.codecProfile} ${encoder.pixelFormat} ${encoder.color} \xB7 rung ${encoder.rung}/${encoder.rungs - 1} cap ${encoder.kbpsCap} kbps${encoder.adaptive ? "" : " pinned"} \xB7 keyframe every ${encoder.keyframeMs} ms \xB7 ${encoder.fps} fps ${encoder.kbps} kbps \xB7 ${encoder.preset}` : "",
+    encoder ? `encoder ${encoder.profile} crf ${encoder.crf} \xB7 ${encoder.codecProfile} ${encoder.pixelFormat} ${encoder.color} \xB7 rung ${encoder.rung}/${encoder.rungs - 1} cap ${encoder.kbpsCap} kbps${encoder.adaptive ? "" : " pinned"} \xB7 keyframe ${encoder.keyframeMs ? `every ${encoder.keyframeMs} ms` : "on demand"} \xB7 ${encoder.fps} fps ${encoder.kbps} kbps \xB7 ${encoder.preset}` : "",
     telemetry && server ? `server cpu: encoder ${telemetry.cpu.encoderPercent}% \xB7 stream ${telemetry.cpu.serverPercent}% \xB7 load ${telemetry.cpu.load} on ${telemetry.cpu.cores} cores \xB7 budget ${server.budgetKbps} kbps sent ${server.sentKbps} \xB7 queue ${server.queueMs} ms ${server.queuedPackets} pkt \xB7 stale ${server.staleEvents} dropped ${server.droppedFrames} \xB7 repairs ${server.repairsSent}/${server.repairsSkipped} skipped \xB7 copies ${server.duplicatesSent}${server.protecting ? " on" : ""} \xB7 remb ${server.rembKbps ?? "\u2014"}` : "",
     transport === "websocket" && server ? `socket: ${Math.round(server.bufferedBytes / 1024)} KiB unsent \xB7 ${server.withheldFrames} deltas withheld${server.waitingForKeyframe ? " \xB7 awaiting keyframe" : ""} \xB7 ${decoderResets} decoder resets` : "",
     telemetry?.events.length ? `server events: ${describeEvents(telemetry.events)}` : "",
@@ -977,6 +1145,8 @@ setInterval(() => {
   renderedFrames = 0;
   decodedAudioPackets = 0;
   renderedWindowStartedAt = now;
+  if (retry) void checkRetry(retry).catch(() => {
+  });
   if (peer) void reportWebRtcStats(peer, stats).catch(() => {
   });
   else if (transport === "websocket") {

package/native/internal/capture/native.go

@@ -152,15 +152,19 @@ func (d *desktop) changed() (bool, error) {
 	return changed, nil
 }
 
-func (d *desktop) wait(micros int64) error {
+func (d *desktop) wait(micros int64, control int) error {
 	C.XFlush(d.display)
-	fd := C.struct_pollfd{fd: C.XConnectionNumber(d.display), events: C.POLLIN}
+	fds := [2]C.struct_pollfd{{fd: C.XConnectionNumber(d.display), events: C.POLLIN}, {fd: C.int(control), events: C.POLLIN}}
+	count := C.nfds_t(1)
+	if control >= 0 {
+		count = 2
+	}
 	timeout := C.struct_timespec{tv_sec: C.time_t(micros / 1000000), tv_nsec: C.long(micros % 1000000 * 1000)}
-	ready, err := C.ppoll(&fd, 1, &timeout, nil)
+	ready, err := C.ppoll(&fds[0], count, &timeout, nil)
 	if ready < 0 && err != syscall.EINTR {
 		return fmt.Errorf("display poll failed: %w", err)
 	}
-	if fd.revents&(C.POLLERR|C.POLLHUP|C.POLLNVAL) != 0 {
+	if fds[0].revents&(C.POLLERR|C.POLLHUP|C.POLLNVAL) != 0 {
 		return errors.New("display disconnected")
 	}
 	return nil
@@ -217,6 +221,7 @@ type encoder struct {
 	packet        *C.AVPacket
 	scale         *C.struct_SwsContext
 	matrix        colorMatrix
+	vbvMS         int
 	headerWritten bool
 }
 
@@ -251,10 +256,19 @@ func newEncoder(o Options) (e *encoder, err error) {
 	}
 	c.time_base = C.AVRational{num: 1, den: 90000}
 	c.framerate = C.AVRational{num: C.int(o.FPS), den: 1}
-	c.gop_size = C.int(max(1, (o.FPS*o.KeyframeMS+500)/1000))
+	keyint := "infinite"
+	c.gop_size = C.int(1 << 30)
+	if o.KeyframeMS > 0 {
+		c.gop_size = C.int(max(1, (o.FPS*o.KeyframeMS+500)/1000))
+		keyint = fmt.Sprintf("%d:min-keyint=%d", c.gop_size, c.gop_size)
+	}
 	c.max_b_frames, c.thread_count = 0, C.int(o.Threads)
+	e.vbvMS = 100
+	if o.KeyframeMS > 0 {
+		e.vbvMS = min(o.KeyframeMS, 100)
+	}
 	c.rc_max_rate = C.int64_t(o.Kbps) * 1000
-	c.rc_buffer_size = C.int(o.Kbps * min(o.KeyframeMS, 100))
+	c.rc_buffer_size = C.int(o.Kbps * e.vbvMS)
 	c.flags |= C.AV_CODEC_FLAG_LOW_DELAY
 	if o.BT709 != 0 {
 		c.color_primaries = C.AVCOL_PRI_BT709
@@ -264,7 +278,7 @@ func newEncoder(o Options) (e *encoder, err error) {
 	}
 	dict := dictionary{}
 	defer dict.close()
-	parameters := fmt.Sprintf("fps=%d/1:repeat-headers=1:sliced-threads=1:slices=4:keyint=%d:min-keyint=%d:scenecut=0:chroma-qp-offset=%d:intra-refresh=%d", o.FPS, c.gop_size, c.gop_size, o.ChromaQP, o.IntraRefresh)
+	parameters := fmt.Sprintf("fps=%d/1:repeat-headers=1:sliced-threads=1:slices=4:keyint=%s:scenecut=0:chroma-qp-offset=%d:intra-refresh=%d", o.FPS, keyint, o.ChromaQP, o.IntraRefresh)
 	if o.Preset == "superfast" {
 		parameters += ":subme=0"
 	}
@@ -366,6 +380,20 @@ func (e *encoder) close() {
 	e.matrix.close()
 }
 
+// reconfigure changes the rate control between frames; libavcodec's x264
+// wrapper applies a changed CRF or VBV with x264_encoder_reconfig before the
+// next frame, so the stream continues without a keyframe.
+func (e *encoder) reconfigure(crf, kbps int) error {
+	name := C.CString("crf")
+	defer C.free(unsafe.Pointer(name))
+	if err := avCheck(C.av_opt_set_double(e.codec.priv_data, name, C.double(crf), 0)); err != nil {
+		return err
+	}
+	e.codec.rc_max_rate = C.int64_t(kbps) * 1000
+	e.codec.rc_buffer_size = C.int(kbps * e.vbvMS)
+	return nil
+}
+
 func (e *encoder) convert(d *desktop, x, y int) error {
 	im := d.image
 	if e.scale != nil {
@@ -442,10 +470,11 @@ func (s *Stream) Close() {
 	runtime.UnlockOSThread()
 }
 
-func NowMicros() int64                                { return int64(C.av_gettime_relative()) }
-func (s *Stream) Changed() (bool, error)              { return s.desktop.changed() }
-func (s *Stream) Wait(micros int64) error             { return s.desktop.wait(micros) }
-func (s *Stream) Send(pts int64, keyframe bool) error { return s.encoder.send(pts, keyframe) }
+func NowMicros() int64                                 { return int64(C.av_gettime_relative()) }
+func (s *Stream) Changed() (bool, error)               { return s.desktop.changed() }
+func (s *Stream) Wait(micros int64, control int) error { return s.desktop.wait(micros, control) }
+func (s *Stream) Send(pts int64, keyframe bool) error  { return s.encoder.send(pts, keyframe) }
+func (s *Stream) Reconfigure(crf, kbps int) error      { return s.encoder.reconfigure(crf, kbps) }
 
 func (s *Stream) Capture(first bool) (copied, converted int64, err error) {
 	if err = avCheck(C.av_frame_make_writable(s.encoder.frame)); err != nil {

package/native/main.go

@@ -1,14 +1,49 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"os/signal"
+	"strconv"
+	"strings"
 	"syscall"
 
 	"github.com/replicas-group/replicas/computer-use/native/internal/capture"
 )
 
+// controls reads the media server's commands from stdin, one per line. They
+// change the running encoder between frames, where a restart would cost a
+// new process and a keyframe. A closed stdin ends control, not capture.
+type controls struct {
+	fd      int
+	pending []byte
+}
+
+func (c *controls) read() []string {
+	var buffer [256]byte
+	for c.fd >= 0 {
+		n, err := syscall.Read(c.fd, buffer[:])
+		if n > 0 {
+			c.pending = append(c.pending, buffer[:n]...)
+			continue
+		}
+		if err == syscall.EINTR {
+			continue
+		}
+		if err != syscall.EAGAIN {
+			c.fd = -1
+		}
+		break
+	}
+	var lines []string
+	for index := bytes.IndexByte(c.pending, '\n'); index >= 0; index = bytes.IndexByte(c.pending, '\n') {
+		lines = append(lines, string(c.pending[:index]))
+		c.pending = c.pending[index+1:]
+	}
+	return lines
+}
+
 func run(o capture.Options) error {
 	stop := make(chan os.Signal, 1)
 	signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
@@ -18,11 +53,15 @@ func run(o capture.Options) error {
 		return err
 	}
 	defer stream.Close()
+	control := &controls{fd: 0}
+	if syscall.SetNonblock(control.fd, true) != nil {
+		control.fd = -1
+	}
 	started := capture.NowMicros()
 	last, lastKeyframe, reportAt := started-1000000, started-1000000, started
 	interval := int64((1000000 + o.FPS - 1) / o.FPS)
 	busyUS, captureUS, scaleUS, encodeUS, previousPTS := int64(0), int64(0), int64(0), int64(0), int64(-1)
-	dirty := true
+	dirty, forceKeyframe := true, false
 	number, changedFrames, sentFrames := 0, 0, 0
 	for {
 		select {
@@ -30,6 +69,27 @@ func run(o capture.Options) error {
 			return nil
 		default:
 		}
+		for _, line := range control.read() {
+			fields := strings.Fields(line)
+			values := make([]int, len(fields))
+			for index := 1; index < len(fields); index++ {
+				if values[index], err = strconv.Atoi(fields[index]); err != nil {
+					values[index] = -1
+				}
+			}
+			switch {
+			case len(fields) == 1 && fields[0] == "keyframe":
+				forceKeyframe = true
+			case len(fields) == 3 && fields[0] == "quality" && values[1] >= 0 && values[1] <= 51 && values[2] > 0:
+				if err = stream.Reconfigure(values[1], values[2]); err != nil {
+					return err
+				}
+			case len(fields) == 2 && fields[0] == "fps" && values[1] > 0 && values[1] <= 240:
+				interval = int64((1000000 + values[1] - 1) / values[1])
+			default:
+				fmt.Fprintf(os.Stderr, "capture: ignoring control %q\n", line)
+			}
+		}
 		changed, err := stream.Changed()
 		if err != nil {
 			return err
@@ -37,15 +97,19 @@ func run(o capture.Options) error {
 		dirty = dirty || changed
 		now := capture.NowMicros()
 		idleDue := last + max(interval, 200000)
-		if o.IntraRefresh == 0 {
+		periodic := o.IntraRefresh == 0 && o.KeyframeMS > 0
+		if periodic {
 			idleDue = min(idleDue, lastKeyframe+int64(o.KeyframeMS)*1000)
 		}
 		due := max(last+interval, idleDue)
 		if dirty || number <= o.Warmup || number < 3 {
 			due = last + interval
 		}
+		if forceKeyframe {
+			due = now
+		}
 		if now < due {
-			if err = stream.Wait(due - now); err != nil {
+			if err = stream.Wait(due-now, control.fd); err != nil {
 				return err
 			}
 			continue
@@ -61,7 +125,8 @@ func run(o capture.Options) error {
 		}
 		pts := max(previousPTS+1, (now-started)*90/1000)
 		previousPTS = pts
-		keyframe := number == 0 || (o.Warmup > 0 && number == o.Warmup) || (o.IntraRefresh == 0 && now-lastKeyframe >= int64(o.KeyframeMS)*1000)
+		keyframe := forceKeyframe || number == 0 || (o.Warmup > 0 && number == o.Warmup) || (periodic && now-lastKeyframe >= int64(o.KeyframeMS)*1000)
+		forceKeyframe = false
 		if keyframe {
 			lastKeyframe = now
 		}

package/native/options.go

@@ -20,7 +20,7 @@ func parseOptions() (capture.Options, error) {
 		{"width", &o.Width, 0, 2, 8192}, {"height", &o.Height, 0, 2, 8192},
 		{"fps", &o.FPS, 60, 1, 240}, {"threads", &o.Threads, 4, 1, 16},
 		{"crf", &o.CRF, 14, 0, 51}, {"kbps", &o.Kbps, 20000, 1, 1000000},
-		{"keyframe-ms", &o.KeyframeMS, 1000, 1, 10000}, {"intra-refresh", &o.IntraRefresh, 1, 0, 1},
+		{"keyframe-ms", &o.KeyframeMS, 1000, 0, 10000}, {"intra-refresh", &o.IntraRefresh, 1, 0, 1},
 		{"chroma-qp", &o.ChromaQP, 0, -12, 12}, {"warmup", &o.Warmup, 0, 0, 60},
 		{"payload", &o.Payload, 96, 0, 127}, {"bt709", &o.BT709, 1, 0, 1},
 	}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-computer",
-  "version": "0.3.13",
+  "version": "0.3.15",
   "description": "Replicas workspace computer-use runtime",
   "type": "module",
   "main": "dist/index.js",