remoterigs-raspberrypi 0.1.14 → 0.1.16

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "remoterigs-raspberrypi",
   "type": "module",
-  "version": "0.1.14",
+  "version": "0.1.16",
   "main": "./index.js",
   "files": [
     "index.js",
@@ -15,6 +15,7 @@
   "dependencies": {
     "@microsoft/signalr": "^8.0.7",
     "axios": "^1.13.2",
+    "node-datachannel": "^0.33.6",
     "pigpio": "^3.3.1"
   }
 }

package/src/common/native/capture-source.js

@@ -0,0 +1,401 @@
+import { spawn, spawnSync } from 'child_process';
+import { EventEmitter } from 'events';
+import dgram from 'dgram';
+import { performance } from 'perf_hooks';
+const START_CODE = Buffer.from([0, 0, 0, 1]);
+const NAL_IDR = 5;
+const NAL_SPS = 7;
+const NAL_PPS = 8;
+const NAL_AUD = 9;
+// wall clock in ms, shared by audio and video so the timestamps of both tracks run on the same clock
+function NowMs() {
+    return performance.timeOrigin + performance.now();
+}
+/**
+ * Captures the webcam (and optionally the microphone) with ffmpeg and emits encoded frames:
+ * - 'video' (VideoFrame): H264, raw Annex-B from ffmpeg stdout, split into access units here
+ * - 'audio' (AudioFrame): Opus, RTP from ffmpeg to a local UDP socket, unpacked here
+ *
+ * One capture per camera, shared by all viewers. The processes only run while there are subscribers.
+ */
+export default class CaptureSource extends EventEmitter {
+    constructor(settings, log) {
+        super();
+        this.settings = settings;
+        this.log = log;
+        this.videoProcess = null;
+        this.audioProcess = null;
+        this.audioSocket = null;
+        this.running = false;
+        this.subscribers = 0;
+        this.stopTimer = null;
+        this.restartTimers = new Set();
+        // mjpeg is the format most USB webcams need for 720p/1080p at 30 fps; not every camera supports it
+        this.useDefaultInputFormat = false;
+        // H264 parsing
+        this.videoBuffer = Buffer.alloc(0);
+        this.currentAccessUnit = [];
+        this.currentAccessUnitTime = 0;
+        this.sps = null;
+        this.pps = null;
+        this.framesEmitted = 0;
+        // audio timestamps: ffmpeg RTP timestamp + offset = wall clock in 48 kHz
+        this.audioTimestampOffset = null;
+        this.resolvedAudioDevice = null;
+        // keep the camera open a bit after the last viewer left, a reconnect (page reload) is then instant
+        this.STOP_DELAY_MS = 15000;
+        this.RESTART_DELAY_MS = 2000;
+        this.setMaxListeners(100);
+    }
+    get HasAudio() {
+        return this.settings.audioPath != "";
+    }
+    Subscribe() {
+        this.subscribers++;
+        if (this.stopTimer != null) {
+            clearTimeout(this.stopTimer);
+            this.stopTimer = null;
+        }
+        if (!this.running) {
+            this.StartCapture();
+        }
+    }
+    Unsubscribe() {
+        this.subscribers = Math.max(0, this.subscribers - 1);
+        if (this.subscribers == 0 && this.running && this.stopTimer == null) {
+            this.stopTimer = setTimeout(() => {
+                this.stopTimer = null;
+                if (this.subscribers == 0) {
+                    this.log("no viewers left, stopping capture");
+                    this.StopCapture();
+                }
+            }, this.STOP_DELAY_MS);
+        }
+    }
+    Dispose() {
+        if (this.stopTimer != null) {
+            clearTimeout(this.stopTimer);
+            this.stopTimer = null;
+        }
+        this.subscribers = 0;
+        this.StopCapture();
+        this.removeAllListeners();
+    }
+    StartCapture() {
+        this.running = true;
+        this.StartVideo();
+        if (this.HasAudio) {
+            this.StartAudio().catch(err => this.log("audio start failed: " + err));
+        }
+    }
+    StopCapture() {
+        this.running = false;
+        for (const timer of this.restartTimers) {
+            clearTimeout(timer);
+        }
+        this.restartTimers.clear();
+        if (this.videoProcess != null) {
+            this.videoProcess.kill("SIGKILL");
+            this.videoProcess = null;
+        }
+        if (this.audioProcess != null) {
+            this.audioProcess.kill("SIGKILL");
+            this.audioProcess = null;
+        }
+        if (this.audioSocket != null) {
+            this.audioSocket.close();
+            this.audioSocket = null;
+        }
+        this.videoBuffer = Buffer.alloc(0);
+        this.currentAccessUnit = [];
+        this.audioTimestampOffset = null;
+    }
+    ScheduleRestart(what, start) {
+        if (!this.running) {
+            return;
+        }
+        this.log(`${what} stopped, restarting in ${this.RESTART_DELAY_MS} ms`);
+        const timer = setTimeout(() => {
+            this.restartTimers.delete(timer);
+            if (this.running) {
+                start();
+            }
+        }, this.RESTART_DELAY_MS);
+        this.restartTimers.add(timer);
+    }
+    // ---------------------------------------------------------------- video
+    BuildVideoArgs() {
+        const s = this.settings;
+        const inputFormat = s.inputFormat != "" ? s.inputFormat : (this.useDefaultInputFormat ? "" : "mjpeg");
+        const args = [
+            "-hide_banner", "-loglevel", "warning", "-nostdin",
+            "-fflags", "nobuffer", "-flags", "low_delay",
+        ];
+        if (s.devicePath.startsWith("lavfi:")) {
+            // test source without a camera, e.g. devicepath = "lavfi:testsrc2"
+            const source = s.devicePath.substring(6);
+            const size = s.width > 0 && s.height > 0 ? `:size=${s.width}x${s.height}` : "";
+            args.push("-re", "-f", "lavfi", "-i", `${source}${source.includes("=") ? ":" : "="}rate=${s.fps}${size}`);
+        }
+        else {
+            args.push("-f", "v4l2", "-thread_queue_size", "64");
+            if (inputFormat != "") {
+                args.push("-input_format", inputFormat);
+            }
+            if (s.width > 0 && s.height > 0) {
+                args.push("-video_size", `${s.width}x${s.height}`);
+            }
+            if (s.fps > 0) {
+                args.push("-framerate", String(s.fps));
+            }
+            args.push("-i", s.devicePath);
+        }
+        args.push("-an");
+        if (inputFormat == "h264") {
+            // the camera encodes itself: no CPU / encoder load at all
+            args.push("-c:v", "copy");
+        }
+        else {
+            args.push("-c:v", s.encoder, "-pix_fmt", "yuv420p", "-b:v", `${s.videoBitrateKbps}k`, "-maxrate", `${s.videoBitrateKbps}k`, "-bufsize", `${s.videoBitrateKbps}k`, 
+            // a short GOP: a viewer that connects (or loses a packet) waits at most this long for a picture
+            "-g", String(s.keyframeIntervalFrames), "-keyint_min", String(s.keyframeIntervalFrames));
+            if (s.encoder == "libx264") {
+                args.push("-preset", "ultrafast", "-tune", "zerolatency", "-profile:v", "baseline");
+            }
+            else {
+                // 66 = H264 baseline (the v4l2m2m wrapper only takes the numeric value)
+                args.push("-profile:v", "66");
+            }
+        }
+        args.push(
+        // an access unit delimiter in front of every frame: makes splitting the stream into frames trivial
+        "-bsf:v", "h264_metadata=aud=insert", "-flush_packets", "1", "-f", "h264", "pipe:1");
+        return args;
+    }
+    StartVideo() {
+        const args = this.BuildVideoArgs();
+        const startedAt = Date.now();
+        let lastStderr = "";
+        this.log("starting video: ffmpeg " + args.join(" "));
+        const proc = spawn("ffmpeg", args, { stdio: ["ignore", "pipe", "pipe"] });
+        this.videoProcess = proc;
+        this.videoBuffer = Buffer.alloc(0);
+        this.currentAccessUnit = [];
+        this.framesEmitted = 0;
+        proc.stdout.on("data", (chunk) => this.OnVideoData(chunk));
+        proc.stderr.on("data", (data) => {
+            lastStderr = data.toString().trim();
+            this.log("ffmpeg video: " + lastStderr);
+        });
+        proc.on("error", (err) => {
+            this.log("ffmpeg video error: " + err.message);
+        });
+        proc.on("exit", (code, signal) => {
+            if (this.videoProcess !== proc) {
+                return;
+            }
+            this.videoProcess = null;
+            this.log(`ffmpeg video exited (code ${code}, signal ${signal}) after ${this.framesEmitted} frames`);
+            // the camera does not do mjpeg: try again with its default format
+            if (this.settings.inputFormat == "" && !this.useDefaultInputFormat && this.framesEmitted == 0 && Date.now() - startedAt < 10000) {
+                this.log("no frames with input format mjpeg, falling back to the camera default format");
+                this.useDefaultInputFormat = true;
+            }
+            this.ScheduleRestart("video", () => this.StartVideo());
+        });
+    }
+    OnVideoData(chunk) {
+        const now = NowMs();
+        const buf = this.videoBuffer.length > 0 ? Buffer.concat([this.videoBuffer, chunk]) : chunk;
+        // find all start codes (00 00 01 or 00 00 00 01)
+        const starts = [];
+        for (let i = 0; i + 2 < buf.length; i++) {
+            if (buf[i] === 0 && buf[i + 1] === 0) {
+                if (buf[i + 2] === 1) {
+                    starts.push({ pos: i, len: 3 });
+                    i += 2;
+                }
+                else if (buf[i + 2] === 0 && i + 3 < buf.length && buf[i + 3] === 1) {
+                    starts.push({ pos: i, len: 4 });
+                    i += 3;
+                }
+            }
+        }
+        if (starts.length == 0) {
+            this.videoBuffer = Buffer.from(buf);
+            return;
+        }
+        // every NAL unit between two start codes is complete; the last one may still grow
+        for (let k = 0; k < starts.length - 1; k++) {
+            const nal = buf.subarray(starts[k].pos + starts[k].len, starts[k + 1].pos);
+            if (nal.length > 0) {
+                this.OnNalUnit(nal, now);
+            }
+        }
+        this.videoBuffer = Buffer.from(buf.subarray(starts[starts.length - 1].pos));
+    }
+    OnNalUnit(nal, now) {
+        const type = nal[0] & 0x1f;
+        if (type == NAL_AUD) {
+            // start of the next frame: the current one is complete
+            if (this.currentAccessUnit.length > 0) {
+                this.EmitAccessUnit();
+            }
+            this.currentAccessUnitTime = now;
+            return;
+        }
+        if (this.currentAccessUnit.length == 0 && this.currentAccessUnitTime == 0) {
+            this.currentAccessUnitTime = now;
+        }
+        if (type == NAL_SPS) {
+            this.sps = Buffer.from(nal);
+        }
+        else if (type == NAL_PPS) {
+            this.pps = Buffer.from(nal);
+        }
+        this.currentAccessUnit.push(nal);
+    }
+    EmitAccessUnit() {
+        const nals = this.currentAccessUnit;
+        this.currentAccessUnit = [];
+        const types = nals.map(n => n[0] & 0x1f);
+        const keyframe = types.includes(NAL_IDR);
+        // the browser can only start decoding at a keyframe WITH SPS/PPS: the encoder does not always repeat them
+        if (keyframe && !types.includes(NAL_SPS) && this.sps != null && this.pps != null) {
+            nals.unshift(this.sps, this.pps);
+        }
+        const parts = [];
+        for (const nal of nals) {
+            parts.push(START_CODE, nal);
+        }
+        const frame = {
+            data: Buffer.concat(parts),
+            timestamp: Math.round(this.currentAccessUnitTime * 90) >>> 0,
+            keyframe: keyframe,
+        };
+        this.framesEmitted++;
+        this.emit("video", frame);
+    }
+    // ---------------------------------------------------------------- audio
+    ResolveAudioDevice(audioPath) {
+        let path = audioPath.trim();
+        // webrtc-streamer style: audiocap://1
+        const capMatch = /^audiocap:\/\/(\d+)$/.exec(path);
+        if (capMatch) {
+            path = capMatch[1];
+        }
+        // ALSA device names (and the lavfi test source) can be used as they are
+        if (/^(lavfi|hw|plughw|default|sysdefault|dsnoop|front):?/.test(path)) {
+            return path;
+        }
+        // a card number
+        if (/^\d+$/.test(path)) {
+            return `plughw:${path},0`;
+        }
+        // a (part of a) card name, e.g. "C920" or "USB Camera"
+        try {
+            const result = spawnSync("arecord", ["-l"], { encoding: "utf8" });
+            const lines = (result.stdout ?? "").split("\n");
+            for (const line of lines) {
+                // card 1: C920 [HD Pro Webcam C920], device 0: USB Audio [USB Audio]
+                const m = /^card (\d+): (\S+) \[([^\]]*)\], device (\d+)/.exec(line);
+                if (m && (line.toLowerCase().includes(path.toLowerCase()))) {
+                    return `plughw:CARD=${m[2]},DEV=${m[4]}`;
+                }
+            }
+        }
+        catch (err) {
+            this.log("arecord -l failed: " + err);
+        }
+        this.log(`audio device '${audioPath}' not found in 'arecord -l', using it as ALSA device name`);
+        return path;
+    }
+    async StartAudio() {
+        if (this.resolvedAudioDevice == null) {
+            this.resolvedAudioDevice = this.ResolveAudioDevice(this.settings.audioPath);
+            this.log(`audio device '${this.settings.audioPath}' -> '${this.resolvedAudioDevice}'`);
+        }
+        if (this.audioSocket == null) {
+            const socket = dgram.createSocket("udp4");
+            socket.on("message", (msg) => this.OnAudioRtp(msg));
+            socket.on("error", (err) => this.log("audio socket error: " + err.message));
+            await new Promise(resolve => socket.bind(0, "127.0.0.1", () => resolve()));
+            this.audioSocket = socket;
+        }
+        if (!this.running) {
+            return;
+        }
+        const port = this.audioSocket.address().port;
+        const args = [
+            "-hide_banner", "-loglevel", "warning", "-nostdin",
+            ...(this.resolvedAudioDevice.startsWith("lavfi:")
+                // test source without a microphone, e.g. audiopath = "lavfi:sine=frequency=440"
+                ? ["-re", "-f", "lavfi", "-i", this.resolvedAudioDevice.substring(6), "-ac", "1", "-ar", "48000"]
+                : ["-f", "alsa", "-thread_queue_size", "1024", "-ac", "1", "-ar", "48000", "-i", this.resolvedAudioDevice]),
+            "-vn",
+            "-c:a", "libopus",
+            "-b:a", `${this.settings.audioBitrateKbps}k`,
+            "-application", "lowdelay",
+            "-frame_duration", "20",
+            "-payload_type", "111",
+            "-f", "rtp",
+            // RTCP to the same socket (we ignore it), not to port+1 which may belong to someone else
+            `rtp://127.0.0.1:${port}?rtcpport=${port}&pkt_size=1200`,
+        ];
+        this.log("starting audio: ffmpeg " + args.join(" "));
+        const proc = spawn("ffmpeg", args, { stdio: ["ignore", "ignore", "pipe"] });
+        this.audioProcess = proc;
+        this.audioTimestampOffset = null;
+        proc.stderr.on("data", (data) => {
+            this.log("ffmpeg audio: " + data.toString().trim());
+        });
+        proc.on("error", (err) => {
+            this.log("ffmpeg audio error: " + err.message);
+        });
+        proc.on("exit", (code, signal) => {
+            if (this.audioProcess !== proc) {
+                return;
+            }
+            this.audioProcess = null;
+            this.log(`ffmpeg audio exited (code ${code}, signal ${signal})`);
+            this.ScheduleRestart("audio", () => { this.StartAudio().catch(err => this.log("audio restart failed: " + err)); });
+        });
+    }
+    OnAudioRtp(packet) {
+        if (packet.length < 12 || (packet[0] >> 6) != 2) {
+            return;
+        }
+        // RTCP (sender reports of ffmpeg): payload type 200..206
+        const payloadTypeByte = packet[1];
+        if (payloadTypeByte >= 200 && payloadTypeByte <= 206) {
+            return;
+        }
+        const padding = (packet[0] >> 5) & 1;
+        const extension = (packet[0] >> 4) & 1;
+        const csrcCount = packet[0] & 0x0f;
+        let offset = 12 + 4 * csrcCount;
+        if (extension) {
+            if (packet.length < offset + 4) {
+                return;
+            }
+            offset += 4 + 4 * packet.readUInt16BE(offset + 2);
+        }
+        let end = packet.length;
+        if (padding) {
+            end -= packet[packet.length - 1];
+        }
+        if (end <= offset) {
+            return;
+        }
+        const rtpTimestamp = packet.readUInt32BE(4);
+        if (this.audioTimestampOffset == null) {
+            this.audioTimestampOffset = Math.round(NowMs() * 48) - rtpTimestamp;
+        }
+        const frame = {
+            data: Buffer.from(packet.subarray(offset, end)),
+            timestamp: (rtpTimestamp + this.audioTimestampOffset) >>> 0,
+        };
+        this.emit("audio", frame);
+    }
+}
+//# sourceMappingURL=capture-source.js.map

package/src/common/native/native-webrtc-streamer.js

@@ -0,0 +1,450 @@
+import nodeDataChannel from 'node-datachannel';
+import { randomInt } from 'crypto';
+import { LogSettingType, VideoType, WebRTCConnectionType, WebRTCMessage } from '../../models/models.js';
+import SignalR from '../signalr.js';
+import Debug from '../debug.js';
+import CaptureSource from './capture-source.js';
+// the payload types match what the relay (SIPSorcery) and the browser already expect from webrtc-streamer
+const H264_PAYLOAD_TYPE = 100;
+const OPUS_PAYLOAD_TYPE = 111;
+// the browser only accepts this H264 variant (setCodecPreferences in WebRTCConnection.ts), the relay uses the same
+const H264_FMTP = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f";
+const VIDEO_MID = "0";
+const AUDIO_MID = "1";
+const STREAM_ID = "remoterigs";
+// the relay on the server (LocalMediaRelay) does not handle audio yet: only send audio on direct connections
+const AUDIO_ON_RELAY = false;
+// how long a peer may stay 'disconnected' before it is closed (libdatachannel can recover from a short drop)
+const DISCONNECTED_TIMEOUT_MS = 15000;
+/**
+ * WebRTC streamer without the webrtc-streamer binary:
+ * - node-datachannel (libdatachannel) is the WebRTC peer (ICE/STUN/TURN, DTLS-SRTP, RTP/RTCP)
+ * - ffmpeg captures and encodes (CaptureSource), h264_v4l2m2m = the hardware encoder of the Pi 4
+ * - the signaling is the same as with WebRTCStreamerWrapper: the rig makes the offer (createOffer),
+ *   the browser / relay answers (setAnswer), candidates are trickled in both directions
+ */
+export default class NativeWebRTCStreamer {
+    static { this.loggerInitialized = false; }
+    constructor(component, clientRTCConfigurationViewModel, videoType) {
+        this.component = component;
+        this.clientRTCConfigurationViewModel = clientRTCConfigurationViewModel;
+        this.videoType = videoType;
+        this.peers = new Map();
+        this.capture = null;
+        this.started = false;
+        this.onVideo = (frame) => this.SendVideo(frame);
+        this.onAudio = (frame) => this.SendAudio(frame);
+    }
+    async Start() {
+        this.component.SetStatusState("State", "Starting");
+        if (!NativeWebRTCStreamer.loggerInitialized) {
+            NativeWebRTCStreamer.loggerInitialized = true;
+            nodeDataChannel.initLogger("Warning", (level, message) => {
+                Debug.LogAlways("libdatachannel", `${level}: ${message}`, false);
+            });
+        }
+        if (this.videoType != VideoType.H264) {
+            this.Log("WARNING: the native streamer only sends H264 (the rig is set to " + VideoType[this.videoType] + ")");
+        }
+        const settings = this.GetCaptureSettings();
+        this.Log("capture settings: " + JSON.stringify(settings));
+        this.capture = new CaptureSource(settings, (message) => this.Log(message));
+        this.capture.on("video", this.onVideo);
+        this.capture.on("audio", this.onAudio);
+        this.started = true;
+        this.component.SetStatusState("State", "Started");
+        this.Log("native streamer is ready (libdatachannel " + nodeDataChannel.getLibraryVersion() + ")");
+    }
+    async Stop() {
+        this.started = false;
+        for (const peer of [...this.peers.values()]) {
+            this.ClosePeer(peer, "streamer stopped");
+        }
+        if (this.capture != null) {
+            this.capture.Dispose();
+            this.capture = null;
+        }
+        this.component.SetStatusState("State", "Stopped");
+    }
+    async PutWebRTCMessage(message) {
+        if (!this.started) {
+            this.Log("IGNORED! PutWebRTCMessage userId: " + message.userId + " messageType: " + message.messageType);
+            return;
+        }
+        this.Log("PutWebRTCMessage userId: " + message.userId + " messageType: " + message.messageType + " session: " + message.sessionId);
+        try {
+            switch (message.messageType) {
+                case "createOffer":
+                    this.CreateOffer(message);
+                    break;
+                case "setAnswer":
+                    this.SetAnswer(message);
+                    break;
+                case "addIceCandidate":
+                    this.AddIceCandidate(message);
+                    break;
+                case "hangup":
+                    this.Hangup(message);
+                    break;
+                case "getIceServers":
+                    this.GetIceServers(message);
+                    break;
+                case "onReceiveCall":
+                    // only used with the 'call' flow of webrtc-streamer
+                    break;
+                case "call":
+                    this.Error("'call' (browser makes the offer) is not supported by the native streamer, use createOffer");
+                    break;
+                default:
+                    this.Log("unknown messageType: " + message.messageType);
+                    break;
+            }
+        }
+        catch (err) {
+            this.Error(`${message.messageType} failed: ${err?.message ?? err}`);
+        }
+    }
+    // ---------------------------------------------------------------- signaling
+    CreateOffer(message) {
+        if (this.capture == null) {
+            throw new Error("capture not started");
+        }
+        // a new offer for an existing session: the browser starts over
+        const existing = this.peers.get(message.sessionId);
+        if (existing != null) {
+            this.ClosePeer(existing, "new offer for the same session");
+        }
+        const pc = new nodeDataChannel.PeerConnection(message.sessionId, {
+            iceServers: this.GetIceServerConfig(),
+            disableAutoNegotiation: true,
+        });
+        // video
+        const videoSsrc = randomInt(1, 0x7fffffff);
+        const video = new nodeDataChannel.Video(VIDEO_MID, "SendOnly");
+        video.addH264Codec(H264_PAYLOAD_TYPE, H264_FMTP);
+        video.addSSRC(videoSsrc, "video", STREAM_ID, "video");
+        const videoTrack = pc.addTrack(video);
+        const videoConfig = new nodeDataChannel.RtpPacketizationConfig(videoSsrc, "video", H264_PAYLOAD_TYPE, 90000);
+        const videoPacketizer = new nodeDataChannel.H264RtpPacketizer("StartSequence", videoConfig);
+        videoPacketizer.addToChain(new nodeDataChannel.RtcpSrReporter(videoConfig));
+        videoPacketizer.addToChain(new nodeDataChannel.RtcpNackResponder());
+        videoTrack.setMediaHandler(videoPacketizer);
+        // audio
+        let audioTrack = null;
+        let audioConfig = null;
+        const withAudio = this.capture.HasAudio && (message.webRTCConnectionType != WebRTCConnectionType.Relay || AUDIO_ON_RELAY);
+        if (withAudio) {
+            const audioSsrc = randomInt(1, 0x7fffffff);
+            const audio = new nodeDataChannel.Audio(AUDIO_MID, "SendOnly");
+            audio.addOpusCodec(OPUS_PAYLOAD_TYPE);
+            audio.addSSRC(audioSsrc, "audio", STREAM_ID, "audio");
+            audioTrack = pc.addTrack(audio);
+            audioConfig = new nodeDataChannel.RtpPacketizationConfig(audioSsrc, "audio", OPUS_PAYLOAD_TYPE, 48000);
+            const audioPacketizer = new nodeDataChannel.RtpPacketizer(audioConfig);
+            audioPacketizer.addToChain(new nodeDataChannel.RtcpSrReporter(audioConfig));
+            audioPacketizer.addToChain(new nodeDataChannel.RtcpNackResponder());
+            audioTrack.setMediaHandler(audioPacketizer);
+        }
+        const peer = {
+            sessionId: message.sessionId,
+            pc: pc,
+            message: message,
+            videoTrack: videoTrack,
+            videoConfig: videoConfig,
+            audioTrack: audioTrack,
+            audioConfig: audioConfig,
+            waitingForKeyframe: true,
+            remoteDescriptionSet: false,
+            pendingRemoteCandidates: [],
+            pendingLocalCandidates: [],
+            disconnectedTimer: null,
+            closed: false,
+            framesSent: 0,
+        };
+        this.peers.set(message.sessionId, peer);
+        pc.onLocalDescription((sdp, type) => {
+            this.Log(`local description (${type}) for session ${peer.sessionId}:\r\n${sdp}`);
+            this.SendToServer(peer, "receiveOffer", JSON.stringify({ type: type, sdp: sdp }));
+        });
+        pc.onLocalCandidate((candidate, mid) => {
+            // like webrtc-streamer: the candidates go out after the answer, the other side then has a peer to add them to
+            if (peer.remoteDescriptionSet) {
+                this.SendLocalCandidate(peer, candidate, mid);
+            }
+            else {
+                peer.pendingLocalCandidates.push({ candidate, mid });
+            }
+        });
+        pc.onStateChange((state) => this.OnPeerStateChange(peer, state));
+        videoTrack.onOpen(() => {
+            this.Log(`video track open for session ${peer.sessionId}, waiting for a keyframe`);
+            peer.waitingForKeyframe = true;
+        });
+        videoTrack.onMessage((msg) => this.OnRtcp(peer, msg));
+        this.capture.Subscribe();
+        this.Log(`creating offer for session ${peer.sessionId} (${WebRTCConnectionType[message.webRTCConnectionType]}, audio: ${withAudio})`);
+        pc.setLocalDescription();
+    }
+    SetAnswer(message) {
+        const peer = this.peers.get(message.sessionId);
+        if (peer == null) {
+            this.Log("setAnswer for unknown session " + message.sessionId);
+            return;
+        }
+        const answer = JSON.parse(message.message);
+        this.Log(`remote description (answer) for session ${peer.sessionId}:\r\n${answer.sdp}`);
+        peer.pc.setRemoteDescription(answer.sdp, "answer");
+        peer.remoteDescriptionSet = true;
+        for (const c of peer.pendingRemoteCandidates) {
+            this.ApplyRemoteCandidate(peer, c.candidate, c.mid);
+        }
+        peer.pendingRemoteCandidates = [];
+        for (const c of peer.pendingLocalCandidates) {
+            this.SendLocalCandidate(peer, c.candidate, c.mid);
+        }
+        peer.pendingLocalCandidates = [];
+    }
+    AddIceCandidate(message) {
+        const peer = this.peers.get(message.sessionId);
+        if (peer == null) {
+            this.Log("addIceCandidate for unknown session " + message.sessionId);
+            return;
+        }
+        const init = JSON.parse(message.message);
+        const candidate = init.candidate ?? "";
+        if (candidate == "") {
+            // end of candidates
+            return;
+        }
+        const mid = init.sdpMid ?? (init.sdpMLineIndex != null ? String(init.sdpMLineIndex) : VIDEO_MID);
+        if (!peer.remoteDescriptionSet) {
+            peer.pendingRemoteCandidates.push({ candidate, mid });
+            return;
+        }
+        this.ApplyRemoteCandidate(peer, candidate, mid);
+    }
+    ApplyRemoteCandidate(peer, candidate, mid) {
+        try {
+            this.Log(`remote candidate for session ${peer.sessionId}: ${candidate}`);
+            peer.pc.addRemoteCandidate(candidate, mid);
+        }
+        catch (err) {
+            // e.g. an mDNS (.local) candidate that cannot be resolved: the other candidates still work
+            this.Log(`addRemoteCandidate failed (${candidate}): ${err?.message ?? err}`);
+        }
+    }
+    SendLocalCandidate(peer, candidate, mid) {
+        this.Log(`local candidate for session ${peer.sessionId}: ${candidate}`);
+        const init = {
+            candidate: candidate.startsWith("a=") ? candidate.substring(2) : candidate,
+            sdpMid: mid,
+            sdpMLineIndex: mid == AUDIO_MID ? 1 : 0,
+        };
+        this.SendToServer(peer, "onReceiveCandidate", JSON.stringify(init));
+    }
+    Hangup(message) {
+        const peer = this.peers.get(message.sessionId);
+        if (peer != null) {
+            this.ClosePeer(peer, "hangup");
+        }
+    }
+    GetIceServers(message) {
+        // same format as /api/getIceServers of webrtc-streamer
+        const iceServers = [];
+        const stun = this.clientRTCConfigurationViewModel.stunServer ?? "";
+        const turn = this.ParseTurnServer();
+        if (stun != "") {
+            iceServers.push({ urls: [`stun:${stun}`] });
+        }
+        if (turn != null) {
+            iceServers.push({ urls: [`turn:${turn.hostname}:${turn.port}`], username: turn.username, credential: turn.password });
+        }
+        const response = new WebRTCMessage();
+        response.componentId = this.component.component.id;
+        response.rigId = message.rigId;
+        response.webRTCConnectionType = message.webRTCConnectionType;
+        response.sessionId = message.sessionId;
+        response.userId = message.userId;
+        response.rigOwnerUserId = message.rigOwnerUserId;
+        response.messageType = "getIceServersResponse";
+        response.message = JSON.stringify({ iceServers });
+        SignalR.SendWebRTCMessage(response);
+    }
+    SendToServer(peer, messageType, payload) {
+        const webrtcMessage = new WebRTCMessage();
+        webrtcMessage.componentId = this.component.component.id;
+        webrtcMessage.rigId = peer.message.rigId;
+        webrtcMessage.webRTCConnectionType = peer.message.webRTCConnectionType;
+        webrtcMessage.sessionId = peer.sessionId;
+        webrtcMessage.userId = peer.message.userId;
+        webrtcMessage.rigOwnerUserId = peer.message.rigOwnerUserId;
+        webrtcMessage.messageType = messageType;
+        webrtcMessage.message = payload;
+        SignalR.SendWebRTCMessage(webrtcMessage);
+    }
+    // ---------------------------------------------------------------- peers
+    OnPeerStateChange(peer, state) {
+        this.Log(`session ${peer.sessionId} state: ${state}`);
+        if (peer.disconnectedTimer != null) {
+            clearTimeout(peer.disconnectedTimer);
+            peer.disconnectedTimer = null;
+        }
+        if (state == "connected") {
+            const pair = peer.pc.getSelectedCandidatePair();
+            if (pair != null) {
+                this.Log(`session ${peer.sessionId} connected via ${pair.local.type}/${pair.local.transportType} -> ${pair.remote.type} ${pair.remote.address}:${pair.remote.port}`);
+            }
+        }
+        else if (state == "disconnected") {
+            peer.disconnectedTimer = setTimeout(() => this.ClosePeer(peer, "disconnected too long"), DISCONNECTED_TIMEOUT_MS);
+        }
+        else if (state == "failed" || state == "closed") {
+            // close() from within the state callback of libdatachannel itself is not allowed: defer it
+            setImmediate(() => this.ClosePeer(peer, state));
+        }
+    }
+    ClosePeer(peer, reason) {
+        if (peer.closed) {
+            return;
+        }
+        peer.closed = true;
+        if (peer.disconnectedTimer != null) {
+            clearTimeout(peer.disconnectedTimer);
+            peer.disconnectedTimer = null;
+        }
+        if (this.peers.get(peer.sessionId) === peer) {
+            this.peers.delete(peer.sessionId);
+        }
+        this.Log(`closing session ${peer.sessionId} (${reason}) after ${peer.framesSent} frames`);
+        try {
+            peer.pc.close();
+        }
+        catch (err) {
+            this.Log("close failed: " + (err?.message ?? err));
+        }
+        this.capture?.Unsubscribe();
+    }
+    OnRtcp(peer, msg) {
+        // walk the compound RTCP packet, looking for PLI (PSFB 206 / FMT 1) and FIR (PSFB 206 / FMT 4)
+        let offset = 0;
+        while (offset + 4 <= msg.length) {
+            const fmt = msg[offset] & 0x1f;
+            const packetType = msg[offset + 1];
+            const length = (msg.readUInt16BE(offset + 2) + 1) * 4;
+            if (packetType == 206 && (fmt == 1 || fmt == 4)) {
+                // the encoder cannot be forced to make a keyframe: the next one (keyframe interval) is at most ~1 s away
+                this.Log(`session ${peer.sessionId} requests a keyframe (${fmt == 1 ? "PLI" : "FIR"})`);
+            }
+            offset += length;
+        }
+    }
+    // ---------------------------------------------------------------- media
+    SendVideo(frame) {
+        for (const peer of this.peers.values()) {
+            if (peer.closed || !peer.videoTrack.isOpen()) {
+                continue;
+            }
+            if (peer.waitingForKeyframe) {
+                if (!frame.keyframe) {
+                    continue;
+                }
+                peer.waitingForKeyframe = false;
+                this.Log(`first keyframe to session ${peer.sessionId} (${frame.data.length} bytes)`);
+            }
+            try {
+                peer.videoConfig.timestamp = frame.timestamp;
+                peer.videoTrack.sendMessageBinary(frame.data);
+                peer.framesSent++;
+            }
+            catch (err) {
+                this.Log(`video send to session ${peer.sessionId} failed: ${err?.message ?? err}`);
+            }
+        }
+    }
+    SendAudio(frame) {
+        for (const peer of this.peers.values()) {
+            if (peer.closed || peer.audioTrack == null || peer.audioConfig == null || !peer.audioTrack.isOpen()) {
+                continue;
+            }
+            try {
+                peer.audioConfig.timestamp = frame.timestamp;
+                peer.audioTrack.sendMessageBinary(frame.data);
+            }
+            catch (err) {
+                this.Log(`audio send to session ${peer.sessionId} failed: ${err?.message ?? err}`);
+            }
+        }
+    }
+    // ---------------------------------------------------------------- configuration
+    GetIceServerConfig() {
+        const servers = [];
+        const stun = this.clientRTCConfigurationViewModel.stunServer ?? "";
+        if (stun != "") {
+            servers.push(`stun:${stun}`);
+        }
+        const turn = this.ParseTurnServer();
+        if (turn != null) {
+            servers.push(turn);
+        }
+        return servers;
+    }
+    // the server sends "username:credential@host:port" (the --turn format of webrtc-streamer);
+    // a TURN REST username contains a ':' itself (expiry:user), the credential (base64) does not
+    ParseTurnServer() {
+        const turn = this.clientRTCConfigurationViewModel.turnServer ?? "";
+        if (turn == "") {
+            return null;
+        }
+        const at = turn.lastIndexOf("@");
+        const hostPort = at >= 0 ? turn.substring(at + 1) : turn;
+        const credentials = at >= 0 ? turn.substring(0, at) : "";
+        const colon = credentials.lastIndexOf(":");
+        const username = colon >= 0 ? credentials.substring(0, colon) : credentials;
+        const password = colon >= 0 ? credentials.substring(colon + 1) : "";
+        const portColon = hostPort.lastIndexOf(":");
+        const hostname = portColon >= 0 ? hostPort.substring(0, portColon) : hostPort;
+        const port = portColon >= 0 ? parseInt(hostPort.substring(portColon + 1)) : 3478;
+        return { hostname, port: isNaN(port) ? 3478 : port, username, password, relayType: "TurnUdp" };
+    }
+    GetCaptureSettings() {
+        const number = (name, fallback) => {
+            const value = parseInt(this.GetOptionalProperty(name));
+            return isNaN(value) || value <= 0 ? fallback : value;
+        };
+        const fps = number("fps", 30);
+        // webrtc-streamer took the bitrate in bps or kbps depending on the version: accept both
+        let bitrate = number("bitrate", 2000);
+        if (bitrate >= 50000) {
+            bitrate = Math.round(bitrate / 1000);
+        }
+        return {
+            devicePath: this.GetOptionalProperty("devicepath"),
+            audioPath: this.GetOptionalProperty("audiopath"),
+            width: number("width", 1280),
+            height: number("height", 720),
+            fps: fps,
+            videoBitrateKbps: bitrate,
+            audioBitrateKbps: number("audiobitrate", 32),
+            inputFormat: this.GetOptionalProperty("inputformat").toLowerCase(),
+            encoder: this.GetOptionalProperty("encoder") || process.env["REMOTERIGS_H264_ENCODER"] || "h264_v4l2m2m",
+            keyframeIntervalFrames: fps,
+        };
+    }
+    // GetProperty logs every missing property: the optional ones are checked here first
+    GetOptionalProperty(name) {
+        if (!this.component.property.hasOwnProperty(name)) {
+            return "";
+        }
+        return (this.component.GetProperty(name) ?? "").trim();
+    }
+    GetName() {
+        return "#" + this.component.component.id + " NativeStreamer";
+    }
+    Log(message) {
+        Debug.Log(LogSettingType.Component, this.component.component.id, this.GetName(), message);
+    }
+    Error(message) {
+        Debug.Error(this.GetName(), message);
+    }
+}
+//# sourceMappingURL=native-webrtc-streamer.js.map

package/src/common/streamer.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streamer.js.map

package/src/common/webrtc-streamer-wrapper.js

@@ -121,10 +121,12 @@ export default class WebRTCStreamerWrapper {
         else {
             this.Log("webrtc-streamer not running, starting it...");
         }
+        const stunServer = this.clientRTCConfigurationViewModel.stunServer ?? "";
         const args = [
             "-H", `0.0.0.0:${this.STREAMER_PORT}`,
-            "-s",
-            //"--stun", this.clientRTCConfigurationViewModel.stunServer,
+            // "-s<host:port>" (no space): our own STUN server, so the rig also has a public (srflx) candidate;
+            // "-s-" = no STUN
+            stunServer != "" ? `-s${stunServer}` : "-s-",
         ];
         if (turnServer != "") {
             args.push("--turn", turnServer);

package/src/component/component.js

@@ -1,5 +1,6 @@
 import Debug from '../common/debug.js';
 import WebRTCStreamerWrapper from "../common/webrtc-streamer-wrapper.js";
+import NativeWebRTCStreamer from "../common/native/native-webrtc-streamer.js";
 import Code from "../common/code.js";
 import StatusNumber from './status/status-number.js';
 import StatusState from './status/status-state.js';
@@ -120,9 +121,17 @@ export default class Component {
             var devicePath = this.GetProperty("devicepath");
             if (devicePath != undefined) {
                 if (this.clientRTCConfigurationViewModel != undefined) {
-                    this.streamer = new WebRTCStreamerWrapper(this, this.clientRTCConfigurationViewModel, this.rig.rig.videoType);
+                    // "native" = node-datachannel + ffmpeg, anything else = the webrtc-streamer binary;
+                    // per camera with the property 'streamer', or for the whole Pi with REMOTERIGS_STREAMER
+                    var streamerType = (this.property.hasOwnProperty("streamer") ? this.GetProperty("streamer") : "") || process.env["REMOTERIGS_STREAMER"] || "webrtc-streamer";
+                    if (streamerType.toLowerCase() == "native") {
+                        this.streamer = new NativeWebRTCStreamer(this, this.clientRTCConfigurationViewModel, this.rig.rig.videoType);
+                    }
+                    else {
+                        this.streamer = new WebRTCStreamerWrapper(this, this.clientRTCConfigurationViewModel, this.rig.rig.videoType);
+                    }
                     this.streamer.Start().catch(console.error);
-                    this.Log("init webcam streamer success: " + devicePath);
+                    this.Log("init webcam streamer (" + streamerType + ") success: " + devicePath);
                 }
                 else {
                     this.Log("init webcam FAILED; missing clientRTCConfigurationViewModel");